├── .gitattributes ├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── app ├── NMF │ ├── Makefile │ ├── script │ │ ├── kill.py │ │ ├── launch.py │ │ ├── launch_on_yarn.py │ │ ├── make_synth_data.py │ │ ├── merge_data.py │ │ ├── partition_data.py │ │ └── run_local.py │ └── src │ │ ├── NMFEngine.cpp │ │ ├── NMFEngine.hpp │ │ ├── matrix_loader.cpp │ │ ├── matrix_loader.hpp │ │ ├── nmf_main.cpp │ │ └── util │ │ ├── Eigen │ │ ├── Array │ │ ├── CMakeLists.txt │ │ ├── Cholesky │ │ ├── CholmodSupport │ │ ├── Core │ │ ├── Dense │ │ ├── Eigen │ │ ├── Eigen2Support │ │ ├── Eigenvalues │ │ ├── Geometry │ │ ├── Householder │ │ ├── IterativeLinearSolvers │ │ ├── Jacobi │ │ ├── LU │ │ ├── LeastSquares │ │ ├── MetisSupport │ │ ├── OrderingMethods │ │ ├── PaStiXSupport │ │ ├── PardisoSupport │ │ ├── QR │ │ ├── QtAlignedMalloc │ │ ├── SPQRSupport │ │ ├── SVD │ │ ├── Sparse │ │ ├── SparseCholesky │ │ ├── SparseCore │ │ ├── SparseLU │ │ ├── SparseQR │ │ ├── StdDeque │ │ ├── StdList │ │ ├── StdVector │ │ ├── SuperLUSupport │ │ ├── UmfPackSupport │ │ └── src │ │ │ ├── CMakeLists.txt │ │ │ ├── Cholesky │ │ │ ├── CMakeLists.txt │ │ │ ├── LDLT.h │ │ │ ├── LLT.h │ │ │ └── LLT_MKL.h │ │ │ ├── CholmodSupport │ │ │ ├── CMakeLists.txt │ │ │ └── CholmodSupport.h │ │ │ ├── Core │ │ │ ├── Array.h │ │ │ ├── ArrayBase.h │ │ │ ├── ArrayWrapper.h │ │ │ ├── Assign.h │ │ │ ├── Assign_MKL.h │ │ │ ├── BandMatrix.h │ │ │ ├── Block.h │ │ │ ├── BooleanRedux.h │ │ │ ├── CMakeLists.txt │ │ │ ├── CommaInitializer.h │ │ │ ├── CoreIterators.h │ │ │ ├── CwiseBinaryOp.h │ │ │ ├── CwiseNullaryOp.h │ │ │ ├── CwiseUnaryOp.h │ │ │ ├── CwiseUnaryView.h │ │ │ ├── DenseBase.h │ │ │ ├── DenseCoeffsBase.h │ │ │ ├── DenseStorage.h │ │ │ ├── Diagonal.h │ │ │ ├── DiagonalMatrix.h │ │ │ ├── DiagonalProduct.h │ │ │ ├── Dot.h │ │ │ ├── EigenBase.h │ │ │ ├── Flagged.h │ │ │ ├── ForceAlignedAccess.h │ │ │ ├── Functors.h │ │ │ ├── Fuzzy.h │ │ │ ├── GeneralProduct.h │ │ │ ├── GenericPacketMath.h │ │ │ ├── GlobalFunctions.h │ │ │ ├── IO.h │ │ │ ├── Map.h │ │ │ ├── MapBase.h │ │ │ ├── MathFunctions.h │ │ │ ├── Matrix.h │ │ │ ├── MatrixBase.h │ │ │ ├── NestByValue.h │ │ │ ├── NoAlias.h │ │ │ ├── NumTraits.h │ │ │ ├── PermutationMatrix.h │ │ │ ├── PlainObjectBase.h │ │ │ ├── ProductBase.h │ │ │ ├── Random.h │ │ │ ├── Redux.h │ │ │ ├── Ref.h │ │ │ ├── Replicate.h │ │ │ ├── ReturnByValue.h │ │ │ ├── Reverse.h │ │ │ ├── Select.h │ │ │ ├── SelfAdjointView.h │ │ │ ├── SelfCwiseBinaryOp.h │ │ │ ├── SolveTriangular.h │ │ │ ├── StableNorm.h │ │ │ ├── Stride.h │ │ │ ├── Swap.h │ │ │ ├── Transpose.h │ │ │ ├── Transpositions.h │ │ │ ├── TriangularMatrix.h │ │ │ ├── VectorBlock.h │ │ │ ├── VectorwiseOp.h │ │ │ ├── Visitor.h │ │ │ ├── arch │ │ │ │ ├── AltiVec │ │ │ │ │ ├── CMakeLists.txt │ │ │ │ │ ├── Complex.h │ │ │ │ │ └── PacketMath.h │ │ │ │ ├── CMakeLists.txt │ │ │ │ ├── Default │ │ │ │ │ ├── CMakeLists.txt │ │ │ │ │ └── Settings.h │ │ │ │ ├── NEON │ │ │ │ │ ├── CMakeLists.txt │ │ │ │ │ ├── Complex.h │ │ │ │ │ └── PacketMath.h │ │ │ │ └── SSE │ │ │ │ │ ├── CMakeLists.txt │ │ │ │ │ ├── Complex.h │ │ │ │ │ ├── MathFunctions.h │ │ │ │ │ └── PacketMath.h │ │ │ ├── products │ │ │ │ ├── CMakeLists.txt │ │ │ │ ├── CoeffBasedProduct.h │ │ │ │ ├── GeneralBlockPanelKernel.h │ │ │ │ ├── GeneralMatrixMatrix.h │ │ │ │ ├── GeneralMatrixMatrixTriangular.h │ │ │ │ ├── GeneralMatrixMatrixTriangular_MKL.h │ │ │ │ ├── GeneralMatrixMatrix_MKL.h │ │ │ │ ├── GeneralMatrixVector.h │ │ │ │ ├── GeneralMatrixVector_MKL.h │ │ │ │ ├── Parallelizer.h │ │ │ │ ├── SelfadjointMatrixMatrix.h │ │ │ │ ├── SelfadjointMatrixMatrix_MKL.h │ │ │ │ ├── SelfadjointMatrixVector.h │ │ │ │ ├── SelfadjointMatrixVector_MKL.h │ │ │ │ ├── SelfadjointProduct.h │ │ │ │ ├── SelfadjointRank2Update.h │ │ │ │ ├── TriangularMatrixMatrix.h │ │ │ │ ├── TriangularMatrixMatrix_MKL.h │ │ │ │ ├── TriangularMatrixVector.h │ │ │ │ ├── TriangularMatrixVector_MKL.h │ │ │ │ ├── TriangularSolverMatrix.h │ │ │ │ ├── TriangularSolverMatrix_MKL.h │ │ │ │ └── TriangularSolverVector.h │ │ │ └── util │ │ │ │ ├── BlasUtil.h │ │ │ │ ├── CMakeLists.txt │ │ │ │ ├── Constants.h │ │ │ │ ├── DisableStupidWarnings.h │ │ │ │ ├── ForwardDeclarations.h │ │ │ │ ├── MKL_support.h │ │ │ │ ├── Macros.h │ │ │ │ ├── Memory.h │ │ │ │ ├── Meta.h │ │ │ │ ├── NonMPL2.h │ │ │ │ ├── ReenableStupidWarnings.h │ │ │ │ ├── StaticAssert.h │ │ │ │ └── XprHelper.h │ │ │ ├── Eigen2Support │ │ │ ├── Block.h │ │ │ ├── CMakeLists.txt │ │ │ ├── Cwise.h │ │ │ ├── CwiseOperators.h │ │ │ ├── Geometry │ │ │ │ ├── AlignedBox.h │ │ │ │ ├── All.h │ │ │ │ ├── AngleAxis.h │ │ │ │ ├── CMakeLists.txt │ │ │ │ ├── Hyperplane.h │ │ │ │ ├── ParametrizedLine.h │ │ │ │ ├── Quaternion.h │ │ │ │ ├── Rotation2D.h │ │ │ │ ├── RotationBase.h │ │ │ │ ├── Scaling.h │ │ │ │ ├── Transform.h │ │ │ │ └── Translation.h │ │ │ ├── LU.h │ │ │ ├── Lazy.h │ │ │ ├── LeastSquares.h │ │ │ ├── Macros.h │ │ │ ├── MathFunctions.h │ │ │ ├── Memory.h │ │ │ ├── Meta.h │ │ │ ├── Minor.h │ │ │ ├── QR.h │ │ │ ├── SVD.h │ │ │ ├── TriangularSolver.h │ │ │ └── VectorBlock.h │ │ │ ├── Eigenvalues │ │ │ ├── CMakeLists.txt │ │ │ ├── ComplexEigenSolver.h │ │ │ ├── ComplexSchur.h │ │ │ ├── ComplexSchur_MKL.h │ │ │ ├── EigenSolver.h │ │ │ ├── GeneralizedEigenSolver.h │ │ │ ├── GeneralizedSelfAdjointEigenSolver.h │ │ │ ├── HessenbergDecomposition.h │ │ │ ├── MatrixBaseEigenvalues.h │ │ │ ├── RealQZ.h │ │ │ ├── RealSchur.h │ │ │ ├── RealSchur_MKL.h │ │ │ ├── SelfAdjointEigenSolver.h │ │ │ ├── SelfAdjointEigenSolver_MKL.h │ │ │ └── Tridiagonalization.h │ │ │ ├── Geometry │ │ │ ├── AlignedBox.h │ │ │ ├── AngleAxis.h │ │ │ ├── CMakeLists.txt │ │ │ ├── EulerAngles.h │ │ │ ├── Homogeneous.h │ │ │ ├── Hyperplane.h │ │ │ ├── OrthoMethods.h │ │ │ ├── ParametrizedLine.h │ │ │ ├── Quaternion.h │ │ │ ├── Rotation2D.h │ │ │ ├── RotationBase.h │ │ │ ├── Scaling.h │ │ │ ├── Transform.h │ │ │ ├── Translation.h │ │ │ ├── Umeyama.h │ │ │ └── arch │ │ │ │ ├── CMakeLists.txt │ │ │ │ └── Geometry_SSE.h │ │ │ ├── Householder │ │ │ ├── BlockHouseholder.h │ │ │ ├── CMakeLists.txt │ │ │ ├── Householder.h │ │ │ └── HouseholderSequence.h │ │ │ ├── IterativeLinearSolvers │ │ │ ├── BasicPreconditioners.h │ │ │ ├── BiCGSTAB.h │ │ │ ├── CMakeLists.txt │ │ │ ├── ConjugateGradient.h │ │ │ ├── IncompleteLUT.h │ │ │ └── IterativeSolverBase.h │ │ │ ├── Jacobi │ │ │ ├── CMakeLists.txt │ │ │ └── Jacobi.h │ │ │ ├── LU │ │ │ ├── CMakeLists.txt │ │ │ ├── Determinant.h │ │ │ ├── FullPivLU.h │ │ │ ├── Inverse.h │ │ │ ├── PartialPivLU.h │ │ │ ├── PartialPivLU_MKL.h │ │ │ └── arch │ │ │ │ ├── CMakeLists.txt │ │ │ │ └── Inverse_SSE.h │ │ │ ├── MetisSupport │ │ │ ├── CMakeLists.txt │ │ │ └── MetisSupport.h │ │ │ ├── OrderingMethods │ │ │ ├── Amd.h │ │ │ ├── CMakeLists.txt │ │ │ ├── Eigen_Colamd.h │ │ │ └── Ordering.h │ │ │ ├── PaStiXSupport │ │ │ ├── CMakeLists.txt │ │ │ └── PaStiXSupport.h │ │ │ ├── PardisoSupport │ │ │ ├── CMakeLists.txt │ │ │ └── PardisoSupport.h │ │ │ ├── QR │ │ │ ├── CMakeLists.txt │ │ │ ├── ColPivHouseholderQR.h │ │ │ ├── ColPivHouseholderQR_MKL.h │ │ │ ├── FullPivHouseholderQR.h │ │ │ ├── HouseholderQR.h │ │ │ └── HouseholderQR_MKL.h │ │ │ ├── SPQRSupport │ │ │ ├── CMakeLists.txt │ │ │ └── SuiteSparseQRSupport.h │ │ │ ├── SVD │ │ │ ├── CMakeLists.txt │ │ │ ├── JacobiSVD.h │ │ │ ├── JacobiSVD_MKL.h │ │ │ └── UpperBidiagonalization.h │ │ │ ├── SparseCholesky │ │ │ ├── CMakeLists.txt │ │ │ ├── SimplicialCholesky.h │ │ │ └── SimplicialCholesky_impl.h │ │ │ ├── SparseCore │ │ │ ├── AmbiVector.h │ │ │ ├── CMakeLists.txt │ │ │ ├── CompressedStorage.h │ │ │ ├── ConservativeSparseSparseProduct.h │ │ │ ├── MappedSparseMatrix.h │ │ │ ├── SparseBlock.h │ │ │ ├── SparseColEtree.h │ │ │ ├── SparseCwiseBinaryOp.h │ │ │ ├── SparseCwiseUnaryOp.h │ │ │ ├── SparseDenseProduct.h │ │ │ ├── SparseDiagonalProduct.h │ │ │ ├── SparseDot.h │ │ │ ├── SparseFuzzy.h │ │ │ ├── SparseMatrix.h │ │ │ ├── SparseMatrixBase.h │ │ │ ├── SparsePermutation.h │ │ │ ├── SparseProduct.h │ │ │ ├── SparseRedux.h │ │ │ ├── SparseSelfAdjointView.h │ │ │ ├── SparseSparseProductWithPruning.h │ │ │ ├── SparseTranspose.h │ │ │ ├── SparseTriangularView.h │ │ │ ├── SparseUtil.h │ │ │ ├── SparseVector.h │ │ │ ├── SparseView.h │ │ │ └── TriangularSolver.h │ │ │ ├── SparseLU │ │ │ ├── CMakeLists.txt │ │ │ ├── SparseLU.h │ │ │ ├── SparseLUImpl.h │ │ │ ├── SparseLU_Memory.h │ │ │ ├── SparseLU_Structs.h │ │ │ ├── SparseLU_SupernodalMatrix.h │ │ │ ├── SparseLU_Utils.h │ │ │ ├── SparseLU_column_bmod.h │ │ │ ├── SparseLU_column_dfs.h │ │ │ ├── SparseLU_copy_to_ucol.h │ │ │ ├── SparseLU_gemm_kernel.h │ │ │ ├── SparseLU_heap_relax_snode.h │ │ │ ├── SparseLU_kernel_bmod.h │ │ │ ├── SparseLU_panel_bmod.h │ │ │ ├── SparseLU_panel_dfs.h │ │ │ ├── SparseLU_pivotL.h │ │ │ ├── SparseLU_pruneL.h │ │ │ └── SparseLU_relax_snode.h │ │ │ ├── SparseQR │ │ │ ├── CMakeLists.txt │ │ │ └── SparseQR.h │ │ │ ├── StlSupport │ │ │ ├── CMakeLists.txt │ │ │ ├── StdDeque.h │ │ │ ├── StdList.h │ │ │ ├── StdVector.h │ │ │ └── details.h │ │ │ ├── SuperLUSupport │ │ │ ├── CMakeLists.txt │ │ │ └── SuperLUSupport.h │ │ │ ├── UmfPackSupport │ │ │ ├── CMakeLists.txt │ │ │ └── UmfPackSupport.h │ │ │ ├── misc │ │ │ ├── CMakeLists.txt │ │ │ ├── Image.h │ │ │ ├── Kernel.h │ │ │ ├── Solve.h │ │ │ ├── SparseSolve.h │ │ │ └── blas.h │ │ │ └── plugins │ │ │ ├── ArrayCwiseBinaryOps.h │ │ │ ├── ArrayCwiseUnaryOps.h │ │ │ ├── BlockMethods.h │ │ │ ├── CMakeLists.txt │ │ │ ├── CommonCwiseBinaryOps.h │ │ │ ├── CommonCwiseUnaryOps.h │ │ │ ├── MatrixCwiseBinaryOps.h │ │ │ └── MatrixCwiseUnaryOps.h │ │ ├── context.cpp │ │ └── context.hpp ├── dml │ ├── Makefile │ ├── datasets │ │ └── dml_para.txt │ ├── script │ │ ├── kill.py │ │ ├── launch.py │ │ ├── launch_on_yarn.py │ │ └── run_local.py │ └── src │ │ └── dml │ │ ├── aux_data.hpp │ │ ├── dml.cpp │ │ ├── dml.hpp │ │ ├── dml_main.cpp │ │ ├── dml_paras.cpp │ │ ├── dml_paras.hpp │ │ ├── types.hpp │ │ ├── utils.cpp │ │ └── utils.hpp ├── dnn │ ├── Makefile │ ├── datasets │ │ └── para_imnet.txt │ ├── script │ │ ├── gen_data.sh │ │ ├── kill.py │ │ ├── launch.py │ │ ├── launch_on_yarn.py │ │ ├── launch_pred.py │ │ ├── predict.py │ │ └── run_local.py │ └── src │ │ ├── dnn │ │ ├── dnn.cpp │ │ ├── dnn.h │ │ ├── dnn_main.cpp │ │ ├── dnn_utils.cpp │ │ ├── dnn_utils.h │ │ ├── paras.cpp │ │ ├── paras.h │ │ ├── types.h │ │ ├── util.cpp │ │ └── util.h │ │ ├── dnn_predict │ │ ├── dnn.cpp │ │ ├── dnn.h │ │ ├── dnn_main.cpp │ │ ├── dnn_utils.cpp │ │ ├── dnn_utils.h │ │ ├── paras.cpp │ │ ├── paras.h │ │ ├── types.h │ │ ├── util.cpp │ │ └── util.h │ │ └── tools │ │ └── gen_data.cpp ├── dnn_speech │ ├── Makefile │ ├── datasets │ │ ├── data_partition.txt │ │ ├── olddata_partition.txt │ │ └── para_imnet.txt │ ├── kaldi-trunk.tar.gz │ ├── machinefiles │ │ └── localserver │ ├── myrun.sh │ ├── petuumdnn │ │ ├── DNN_config.cc │ │ ├── Makefile │ │ ├── ali_count.cc │ │ ├── ark2txt.cc │ │ ├── asc2bin.cc │ │ ├── bin2asc.cc │ │ ├── check.cc │ │ ├── egs2txt.cc │ │ ├── feature_extraction.cc │ │ ├── parse_head.cc │ │ ├── set_num_layers.cc │ │ ├── testcatright.cc │ │ └── tot_extraction.cc │ ├── scripts │ │ ├── NetworkDecode.sh │ │ ├── PrepDNNFeature.sh │ │ ├── adjust_prior.sh │ │ ├── gen_data.sh │ │ ├── kill_dnn.sh │ │ ├── kill_dnn_sn.sh │ │ ├── path.sh │ │ ├── petuum_dnn.sh │ │ ├── run_dnn.sh │ │ ├── run_dnn_sn.sh │ │ ├── timit.sh │ │ ├── timit_data_prep.sh │ │ └── timit_decode.sh │ └── src │ │ ├── dnn │ │ ├── dnn.cpp │ │ ├── dnn.h │ │ ├── dnn_main.cpp │ │ ├── dnn_utils.cpp │ │ ├── dnn_utils.h │ │ ├── paras.cpp │ │ ├── paras.h │ │ ├── types.h │ │ ├── util.cpp │ │ └── util.h │ │ └── tools │ │ └── gen_data.cpp ├── kmeans │ ├── Makefile │ ├── README.md │ ├── dataset │ │ ├── data_generate.py │ │ └── sample.txt │ ├── script │ │ ├── kill.py │ │ ├── launch.py.template │ │ ├── launch_on_yarn.py │ │ └── run_local.py.template │ └── src │ │ ├── cluster_centers.cpp │ │ ├── cluster_centers.h │ │ ├── context.cpp │ │ ├── context.hpp │ │ ├── dataset.cpp │ │ ├── dataset.h │ │ ├── dense_vector.cpp │ │ ├── dense_vector.h │ │ ├── kmeans.cpp │ │ ├── kmeans.h │ │ ├── kmeans_main.cpp │ │ ├── kmeans_methods.cpp │ │ ├── kmeans_methods.h │ │ ├── kmeans_worker.cpp │ │ ├── kmeans_worker.h │ │ ├── sparse_vector.cpp │ │ └── sparse_vector.h ├── mlr │ ├── Makefile │ ├── datasets │ │ ├── covtype.scale.test.small │ │ ├── covtype.scale.test.small.meta │ │ ├── covtype.scale.train.small │ │ └── covtype.scale.train.small.meta │ ├── script │ │ ├── kill.py │ │ ├── launch.py.template │ │ ├── launch_on_yarn.py │ │ ├── run_gen_data.py │ │ └── run_local.py.template │ └── src │ │ ├── abstract_mlr_sgd_solver.hpp │ │ ├── common.hpp │ │ ├── lr_sgd_solver.cpp │ │ ├── lr_sgd_solver.hpp │ │ ├── mlr_engine.cpp │ │ ├── mlr_engine.hpp │ │ ├── mlr_main.cpp │ │ ├── mlr_sgd_solver.cpp │ │ ├── mlr_sgd_solver.hpp │ │ └── tools │ │ ├── gen_data_sparse.cpp │ │ └── gen_data_sparse.cpp.bak ├── rand_forest │ ├── Makefile │ ├── ReadMe.md │ ├── dataset │ │ ├── forest.model │ │ ├── iris.test │ │ ├── iris.test.meta │ │ ├── iris.train │ │ └── iris.train.meta │ ├── script │ │ ├── kill.py │ │ ├── launch.py.template │ │ ├── launch_on_yarn.py │ │ └── run_local.py.template │ └── src │ │ ├── common.hpp │ │ ├── decision_tree.cpp │ │ ├── decision_tree.hpp │ │ ├── rand_forest.cpp │ │ ├── rand_forest.hpp │ │ ├── rand_forest_engine.cpp │ │ ├── rand_forest_engine.hpp │ │ ├── rand_forest_main.cpp │ │ ├── split_finder.cpp │ │ ├── split_finder.hpp │ │ ├── tree_node.hpp │ │ ├── utils.cpp │ │ └── utils.hpp └── sparsecoding │ ├── Makefile │ ├── script │ ├── kill.py │ ├── launch.py │ ├── launch_on_yarn.py │ ├── make_synth_data.py │ ├── merge_data.py │ ├── partition_data.py │ └── run_local.py │ └── src │ ├── SCEngine.cpp │ ├── SCEngine.hpp │ ├── matrix_loader.cpp │ ├── matrix_loader.hpp │ ├── sparsecoding_main.cpp │ └── util │ ├── Eigen │ ├── Array │ ├── CMakeLists.txt │ ├── Cholesky │ ├── CholmodSupport │ ├── Core │ ├── Dense │ ├── Eigen │ ├── Eigen2Support │ ├── Eigenvalues │ ├── Geometry │ ├── Householder │ ├── IterativeLinearSolvers │ ├── Jacobi │ ├── LU │ ├── LeastSquares │ ├── MetisSupport │ ├── OrderingMethods │ ├── PaStiXSupport │ ├── PardisoSupport │ ├── QR │ ├── QtAlignedMalloc │ ├── SPQRSupport │ ├── SVD │ ├── Sparse │ ├── SparseCholesky │ ├── SparseCore │ ├── SparseLU │ ├── SparseQR │ ├── StdDeque │ ├── StdList │ ├── StdVector │ ├── SuperLUSupport │ ├── UmfPackSupport │ └── src │ │ ├── CMakeLists.txt │ │ ├── Cholesky │ │ ├── CMakeLists.txt │ │ ├── LDLT.h │ │ ├── LLT.h │ │ └── LLT_MKL.h │ │ ├── CholmodSupport │ │ ├── CMakeLists.txt │ │ └── CholmodSupport.h │ │ ├── Core │ │ ├── Array.h │ │ ├── ArrayBase.h │ │ ├── ArrayWrapper.h │ │ ├── Assign.h │ │ ├── Assign_MKL.h │ │ ├── BandMatrix.h │ │ ├── Block.h │ │ ├── BooleanRedux.h │ │ ├── CMakeLists.txt │ │ ├── CommaInitializer.h │ │ ├── CoreIterators.h │ │ ├── CwiseBinaryOp.h │ │ ├── CwiseNullaryOp.h │ │ ├── CwiseUnaryOp.h │ │ ├── CwiseUnaryView.h │ │ ├── DenseBase.h │ │ ├── DenseCoeffsBase.h │ │ ├── DenseStorage.h │ │ ├── Diagonal.h │ │ ├── DiagonalMatrix.h │ │ ├── DiagonalProduct.h │ │ ├── Dot.h │ │ ├── EigenBase.h │ │ ├── Flagged.h │ │ ├── ForceAlignedAccess.h │ │ ├── Functors.h │ │ ├── Fuzzy.h │ │ ├── GeneralProduct.h │ │ ├── GenericPacketMath.h │ │ ├── GlobalFunctions.h │ │ ├── IO.h │ │ ├── Map.h │ │ ├── MapBase.h │ │ ├── MathFunctions.h │ │ ├── Matrix.h │ │ ├── MatrixBase.h │ │ ├── NestByValue.h │ │ ├── NoAlias.h │ │ ├── NumTraits.h │ │ ├── PermutationMatrix.h │ │ ├── PlainObjectBase.h │ │ ├── ProductBase.h │ │ ├── Random.h │ │ ├── Redux.h │ │ ├── Ref.h │ │ ├── Replicate.h │ │ ├── ReturnByValue.h │ │ ├── Reverse.h │ │ ├── Select.h │ │ ├── SelfAdjointView.h │ │ ├── SelfCwiseBinaryOp.h │ │ ├── SolveTriangular.h │ │ ├── StableNorm.h │ │ ├── Stride.h │ │ ├── Swap.h │ │ ├── Transpose.h │ │ ├── Transpositions.h │ │ ├── TriangularMatrix.h │ │ ├── VectorBlock.h │ │ ├── VectorwiseOp.h │ │ ├── Visitor.h │ │ ├── arch │ │ │ ├── AltiVec │ │ │ │ ├── CMakeLists.txt │ │ │ │ ├── Complex.h │ │ │ │ └── PacketMath.h │ │ │ ├── CMakeLists.txt │ │ │ ├── Default │ │ │ │ ├── CMakeLists.txt │ │ │ │ └── Settings.h │ │ │ ├── NEON │ │ │ │ ├── CMakeLists.txt │ │ │ │ ├── Complex.h │ │ │ │ └── PacketMath.h │ │ │ └── SSE │ │ │ │ ├── CMakeLists.txt │ │ │ │ ├── Complex.h │ │ │ │ ├── MathFunctions.h │ │ │ │ └── PacketMath.h │ │ ├── products │ │ │ ├── CMakeLists.txt │ │ │ ├── CoeffBasedProduct.h │ │ │ ├── GeneralBlockPanelKernel.h │ │ │ ├── GeneralMatrixMatrix.h │ │ │ ├── GeneralMatrixMatrixTriangular.h │ │ │ ├── GeneralMatrixMatrixTriangular_MKL.h │ │ │ ├── GeneralMatrixMatrix_MKL.h │ │ │ ├── GeneralMatrixVector.h │ │ │ ├── GeneralMatrixVector_MKL.h │ │ │ ├── Parallelizer.h │ │ │ ├── SelfadjointMatrixMatrix.h │ │ │ ├── SelfadjointMatrixMatrix_MKL.h │ │ │ ├── SelfadjointMatrixVector.h │ │ │ ├── SelfadjointMatrixVector_MKL.h │ │ │ ├── SelfadjointProduct.h │ │ │ ├── SelfadjointRank2Update.h │ │ │ ├── TriangularMatrixMatrix.h │ │ │ ├── TriangularMatrixMatrix_MKL.h │ │ │ ├── TriangularMatrixVector.h │ │ │ ├── TriangularMatrixVector_MKL.h │ │ │ ├── TriangularSolverMatrix.h │ │ │ ├── TriangularSolverMatrix_MKL.h │ │ │ └── TriangularSolverVector.h │ │ └── util │ │ │ ├── BlasUtil.h │ │ │ ├── CMakeLists.txt │ │ │ ├── Constants.h │ │ │ ├── DisableStupidWarnings.h │ │ │ ├── ForwardDeclarations.h │ │ │ ├── MKL_support.h │ │ │ ├── Macros.h │ │ │ ├── Memory.h │ │ │ ├── Meta.h │ │ │ ├── NonMPL2.h │ │ │ ├── ReenableStupidWarnings.h │ │ │ ├── StaticAssert.h │ │ │ └── XprHelper.h │ │ ├── Eigen2Support │ │ ├── Block.h │ │ ├── CMakeLists.txt │ │ ├── Cwise.h │ │ ├── CwiseOperators.h │ │ ├── Geometry │ │ │ ├── AlignedBox.h │ │ │ ├── All.h │ │ │ ├── AngleAxis.h │ │ │ ├── CMakeLists.txt │ │ │ ├── Hyperplane.h │ │ │ ├── ParametrizedLine.h │ │ │ ├── Quaternion.h │ │ │ ├── Rotation2D.h │ │ │ ├── RotationBase.h │ │ │ ├── Scaling.h │ │ │ ├── Transform.h │ │ │ └── Translation.h │ │ ├── LU.h │ │ ├── Lazy.h │ │ ├── LeastSquares.h │ │ ├── Macros.h │ │ ├── MathFunctions.h │ │ ├── Memory.h │ │ ├── Meta.h │ │ ├── Minor.h │ │ ├── QR.h │ │ ├── SVD.h │ │ ├── TriangularSolver.h │ │ └── VectorBlock.h │ │ ├── Eigenvalues │ │ ├── CMakeLists.txt │ │ ├── ComplexEigenSolver.h │ │ ├── ComplexSchur.h │ │ ├── ComplexSchur_MKL.h │ │ ├── EigenSolver.h │ │ ├── GeneralizedEigenSolver.h │ │ ├── GeneralizedSelfAdjointEigenSolver.h │ │ ├── HessenbergDecomposition.h │ │ ├── MatrixBaseEigenvalues.h │ │ ├── RealQZ.h │ │ ├── RealSchur.h │ │ ├── RealSchur_MKL.h │ │ ├── SelfAdjointEigenSolver.h │ │ ├── SelfAdjointEigenSolver_MKL.h │ │ └── Tridiagonalization.h │ │ ├── Geometry │ │ ├── AlignedBox.h │ │ ├── AngleAxis.h │ │ ├── CMakeLists.txt │ │ ├── EulerAngles.h │ │ ├── Homogeneous.h │ │ ├── Hyperplane.h │ │ ├── OrthoMethods.h │ │ ├── ParametrizedLine.h │ │ ├── Quaternion.h │ │ ├── Rotation2D.h │ │ ├── RotationBase.h │ │ ├── Scaling.h │ │ ├── Transform.h │ │ ├── Translation.h │ │ ├── Umeyama.h │ │ └── arch │ │ │ ├── CMakeLists.txt │ │ │ └── Geometry_SSE.h │ │ ├── Householder │ │ ├── BlockHouseholder.h │ │ ├── CMakeLists.txt │ │ ├── Householder.h │ │ └── HouseholderSequence.h │ │ ├── IterativeLinearSolvers │ │ ├── BasicPreconditioners.h │ │ ├── BiCGSTAB.h │ │ ├── CMakeLists.txt │ │ ├── ConjugateGradient.h │ │ ├── IncompleteLUT.h │ │ └── IterativeSolverBase.h │ │ ├── Jacobi │ │ ├── CMakeLists.txt │ │ └── Jacobi.h │ │ ├── LU │ │ ├── CMakeLists.txt │ │ ├── Determinant.h │ │ ├── FullPivLU.h │ │ ├── Inverse.h │ │ ├── PartialPivLU.h │ │ ├── PartialPivLU_MKL.h │ │ └── arch │ │ │ ├── CMakeLists.txt │ │ │ └── Inverse_SSE.h │ │ ├── MetisSupport │ │ ├── CMakeLists.txt │ │ └── MetisSupport.h │ │ ├── OrderingMethods │ │ ├── Amd.h │ │ ├── CMakeLists.txt │ │ ├── Eigen_Colamd.h │ │ └── Ordering.h │ │ ├── PaStiXSupport │ │ ├── CMakeLists.txt │ │ └── PaStiXSupport.h │ │ ├── PardisoSupport │ │ ├── CMakeLists.txt │ │ └── PardisoSupport.h │ │ ├── QR │ │ ├── CMakeLists.txt │ │ ├── ColPivHouseholderQR.h │ │ ├── ColPivHouseholderQR_MKL.h │ │ ├── FullPivHouseholderQR.h │ │ ├── HouseholderQR.h │ │ └── HouseholderQR_MKL.h │ │ ├── SPQRSupport │ │ ├── CMakeLists.txt │ │ └── SuiteSparseQRSupport.h │ │ ├── SVD │ │ ├── CMakeLists.txt │ │ ├── JacobiSVD.h │ │ ├── JacobiSVD_MKL.h │ │ └── UpperBidiagonalization.h │ │ ├── SparseCholesky │ │ ├── CMakeLists.txt │ │ ├── SimplicialCholesky.h │ │ └── SimplicialCholesky_impl.h │ │ ├── SparseCore │ │ ├── AmbiVector.h │ │ ├── CMakeLists.txt │ │ ├── CompressedStorage.h │ │ ├── ConservativeSparseSparseProduct.h │ │ ├── MappedSparseMatrix.h │ │ ├── SparseBlock.h │ │ ├── SparseColEtree.h │ │ ├── SparseCwiseBinaryOp.h │ │ ├── SparseCwiseUnaryOp.h │ │ ├── SparseDenseProduct.h │ │ ├── SparseDiagonalProduct.h │ │ ├── SparseDot.h │ │ ├── SparseFuzzy.h │ │ ├── SparseMatrix.h │ │ ├── SparseMatrixBase.h │ │ ├── SparsePermutation.h │ │ ├── SparseProduct.h │ │ ├── SparseRedux.h │ │ ├── SparseSelfAdjointView.h │ │ ├── SparseSparseProductWithPruning.h │ │ ├── SparseTranspose.h │ │ ├── SparseTriangularView.h │ │ ├── SparseUtil.h │ │ ├── SparseVector.h │ │ ├── SparseView.h │ │ └── TriangularSolver.h │ │ ├── SparseLU │ │ ├── CMakeLists.txt │ │ ├── SparseLU.h │ │ ├── SparseLUImpl.h │ │ ├── SparseLU_Memory.h │ │ ├── SparseLU_Structs.h │ │ ├── SparseLU_SupernodalMatrix.h │ │ ├── SparseLU_Utils.h │ │ ├── SparseLU_column_bmod.h │ │ ├── SparseLU_column_dfs.h │ │ ├── SparseLU_copy_to_ucol.h │ │ ├── SparseLU_gemm_kernel.h │ │ ├── SparseLU_heap_relax_snode.h │ │ ├── SparseLU_kernel_bmod.h │ │ ├── SparseLU_panel_bmod.h │ │ ├── SparseLU_panel_dfs.h │ │ ├── SparseLU_pivotL.h │ │ ├── SparseLU_pruneL.h │ │ └── SparseLU_relax_snode.h │ │ ├── SparseQR │ │ ├── CMakeLists.txt │ │ └── SparseQR.h │ │ ├── StlSupport │ │ ├── CMakeLists.txt │ │ ├── StdDeque.h │ │ ├── StdList.h │ │ ├── StdVector.h │ │ └── details.h │ │ ├── SuperLUSupport │ │ ├── CMakeLists.txt │ │ └── SuperLUSupport.h │ │ ├── UmfPackSupport │ │ ├── CMakeLists.txt │ │ └── UmfPackSupport.h │ │ ├── misc │ │ ├── CMakeLists.txt │ │ ├── Image.h │ │ ├── Kernel.h │ │ ├── Solve.h │ │ ├── SparseSolve.h │ │ └── blas.h │ │ └── plugins │ │ ├── ArrayCwiseBinaryOps.h │ │ ├── ArrayCwiseUnaryOps.h │ │ ├── BlockMethods.h │ │ ├── CMakeLists.txt │ │ ├── CommonCwiseBinaryOps.h │ │ ├── CommonCwiseUnaryOps.h │ │ ├── MatrixCwiseBinaryOps.h │ │ └── MatrixCwiseUnaryOps.h │ ├── context.cpp │ └── context.hpp ├── defns.mk.template ├── docs └── bosen_refman.pdf ├── machinefiles └── localserver ├── src ├── io │ ├── general_fstream.cpp │ ├── general_fstream.hpp │ ├── hdfs.cpp │ └── hdfs.hpp ├── ml │ ├── feature │ │ ├── abstract_datum.hpp │ │ ├── abstract_feature.hpp │ │ ├── dense_feature.hpp │ │ └── sparse_feature.hpp │ ├── include │ │ └── ml.hpp │ └── util │ │ ├── data_loading.cpp │ │ ├── data_loading.hpp │ │ ├── fastapprox │ │ ├── cast.h │ │ ├── fastapprox.hpp │ │ ├── fastexp.h │ │ ├── fastgamma.h │ │ ├── fasthyperbolic.h │ │ ├── fastlog.h │ │ ├── fastpow.h │ │ ├── fastsigmoid.h │ │ └── sse.h │ │ ├── math_util.cpp │ │ ├── math_util.hpp │ │ ├── metafile_reader.cpp │ │ ├── metafile_reader.hpp │ │ └── workload_manager.hpp ├── petuum.mk ├── petuum_ps │ ├── client │ │ ├── client_table.cpp │ │ ├── client_table.hpp │ │ ├── oplog_serializer.hpp │ │ ├── serialized_row_reader.hpp │ │ ├── ssp_client_row.hpp │ │ ├── table_group.cpp │ │ ├── table_group.hpp │ │ ├── thread_table.cpp │ │ └── thread_table.hpp │ ├── consistency │ │ ├── ssp_aggr_consistency_controller.cpp │ │ ├── ssp_aggr_consistency_controller.hpp │ │ ├── ssp_aggr_value_consistency_controller.cpp │ │ ├── ssp_aggr_value_consistency_controller.hpp │ │ ├── ssp_consistency_controller.cpp │ │ ├── ssp_consistency_controller.hpp │ │ ├── ssp_push_append_only_consistency_controller.cpp │ │ ├── ssp_push_append_only_consistency_controller.hpp │ │ ├── ssp_push_consistency_controller.cpp │ │ └── ssp_push_consistency_controller.hpp │ ├── oplog │ │ ├── abstract_oplog.hpp │ │ ├── append_only_oplog.hpp │ │ ├── append_only_oplog_partition.cpp │ │ ├── append_only_oplog_partition.hpp │ │ ├── create_row_oplog.cpp │ │ ├── create_row_oplog.hpp │ │ ├── dense_oplog.cpp │ │ ├── dense_oplog.hpp │ │ ├── meta_row_oplog.hpp │ │ ├── oplog_index.cpp │ │ ├── oplog_index.hpp │ │ ├── oplog_partition.hpp │ │ ├── row_oplog_meta.hpp │ │ ├── sparse_oplog.cpp │ │ └── sparse_oplog.hpp │ ├── server │ │ ├── callback_subs.hpp │ │ ├── name_node.cpp │ │ ├── name_node.hpp │ │ ├── name_node_thread.cpp │ │ ├── name_node_thread.hpp │ │ ├── serialized_oplog_reader.hpp │ │ ├── server.cpp │ │ ├── server.hpp │ │ ├── server_row.hpp │ │ ├── server_table.cpp │ │ ├── server_table.hpp │ │ ├── server_thread.cpp │ │ ├── server_thread.hpp │ │ ├── server_thread_group.cpp │ │ ├── server_thread_group.hpp │ │ ├── server_threads.cpp │ │ ├── server_threads.hpp │ │ ├── ssp_aggr_server_thread.cpp │ │ ├── ssp_aggr_server_thread.hpp │ │ ├── ssp_push_server_thread.cpp │ │ └── ssp_push_server_thread.hpp │ └── thread │ │ ├── abstract_bg_worker.cpp │ │ ├── abstract_bg_worker.hpp │ │ ├── append_only_row_oplog_buffer.cpp │ │ ├── append_only_row_oplog_buffer.hpp │ │ ├── bg_oplog.hpp │ │ ├── bg_oplog_partition.cpp │ │ ├── bg_oplog_partition.hpp │ │ ├── bg_worker_group.cpp │ │ ├── bg_worker_group.hpp │ │ ├── bg_workers.cpp │ │ ├── bg_workers.hpp │ │ ├── context.cpp │ │ ├── context.hpp │ │ ├── msg_tracker.cpp │ │ ├── msg_tracker.hpp │ │ ├── oplog_meta.hpp │ │ ├── ps_msgs.hpp │ │ ├── row_oplog_recycle.hpp │ │ ├── row_oplog_serializer.cpp │ │ ├── row_oplog_serializer.hpp │ │ ├── row_request_oplog_mgr.cpp │ │ ├── row_request_oplog_mgr.hpp │ │ ├── server_version_mgr.cpp │ │ ├── server_version_mgr.hpp │ │ ├── ssp_aggr_bg_worker.cpp │ │ ├── ssp_aggr_bg_worker.hpp │ │ ├── ssp_bg_worker.cpp │ │ ├── ssp_bg_worker.hpp │ │ ├── ssp_push_bg_worker.cpp │ │ ├── ssp_push_bg_worker.hpp │ │ ├── ssp_push_bg_worker_group.cpp │ │ ├── ssp_push_bg_worker_group.hpp │ │ ├── ssp_push_row_request_oplog_mgr.cpp │ │ ├── ssp_push_row_request_oplog_mgr.hpp │ │ ├── table_oplog_meta.cpp │ │ ├── table_oplog_meta.hpp │ │ └── trans_time_estimate.hpp ├── petuum_ps_common │ ├── client │ │ ├── abstract_client_table.hpp │ │ ├── abstract_table_group.hpp │ │ └── client_row.hpp │ ├── comm_bus │ │ ├── comm_bus.cpp │ │ ├── comm_bus.hpp │ │ ├── zmq_util.cpp │ │ └── zmq_util.hpp │ ├── consistency │ │ └── abstract_consistency_controller.hpp │ ├── include │ │ ├── abstract_row.hpp │ │ ├── configs.hpp │ │ ├── constants.hpp │ │ ├── host_info.hpp │ │ ├── petuum_ps.hpp │ │ ├── ps_table_group.cpp │ │ ├── ps_table_group.hpp │ │ ├── row_access.hpp │ │ ├── system_gflags.hpp │ │ ├── table.hpp │ │ └── table_gflags.hpp │ ├── oplog │ │ ├── abstract_append_only_buffer.hpp │ │ ├── abstract_row_oplog.hpp │ │ ├── batch_inc_append_only_buffer.cpp │ │ ├── batch_inc_append_only_buffer.hpp │ │ ├── buffer_manager.hpp │ │ ├── dense_append_only_buffer.cpp │ │ ├── dense_append_only_buffer.hpp │ │ ├── dense_row_oplog.hpp │ │ ├── inc_append_only_buffer.cpp │ │ ├── inc_append_only_buffer.hpp │ │ ├── sparse_row_oplog.hpp │ │ └── sparse_vector_row_oplog.hpp │ ├── storage │ │ ├── abstract_process_storage.hpp │ │ ├── bounded_dense_process_storage.cpp │ │ ├── bounded_dense_process_storage.hpp │ │ ├── bounded_sparse_process_storage.cpp │ │ ├── bounded_sparse_process_storage.hpp │ │ ├── clock_lru.cpp │ │ ├── clock_lru.hpp │ │ ├── dense_row.hpp │ │ ├── entry.hpp │ │ ├── multiplicative_dense_row.hpp │ │ ├── numeric_container_row.hpp │ │ ├── sorted_vector_map_row.hpp │ │ ├── sparse_feature_row.hpp │ │ └── sparse_row.hpp │ ├── thread │ │ ├── mem_transfer.hpp │ │ └── msg_base.hpp │ └── util │ │ ├── class_register.hpp │ │ ├── high_resolution_timer.cpp │ │ ├── high_resolution_timer.hpp │ │ ├── lock.cpp │ │ ├── lock.hpp │ │ ├── lockable.hpp │ │ ├── mem_block.hpp │ │ ├── mpmc_queue.hpp │ │ ├── mt_queue.hpp │ │ ├── record_buff.hpp │ │ ├── sparse_vector.cpp │ │ ├── sparse_vector.hpp │ │ ├── sparse_vector_mgr.hpp │ │ ├── stats.cpp │ │ ├── stats.hpp │ │ ├── striped_lock.hpp │ │ ├── thread.hpp │ │ ├── thread_group.hpp │ │ ├── timer_thr.cpp │ │ ├── timer_thr.hpp │ │ ├── utils.cpp │ │ ├── utils.hpp │ │ ├── vector_clock.cpp │ │ ├── vector_clock.hpp │ │ ├── vector_clock_mt.cpp │ │ ├── vector_clock_mt.hpp │ │ └── vector_map.hpp └── yarn │ ├── README.md │ ├── build.gradle │ ├── gradle.properties.template │ └── src │ ├── main │ └── java │ │ └── cn │ │ └── edu │ │ └── buaa │ │ └── act │ │ └── petuumOnYarn │ │ ├── ApplicationMaster.java │ │ ├── Client.java │ │ ├── Log4jPropertyHelper.java │ │ └── YarnUtil.java │ └── test │ ├── java │ ├── cn │ │ └── edu │ │ │ └── buaa │ │ │ └── act │ │ │ └── petuumOnYarn │ │ │ └── TestPetuumOnYarn.java │ └── org │ │ └── apache │ │ └── hadoop │ │ └── util │ │ └── JarFinder.java │ └── resources │ └── log4j.properties └── style_script └── cpplint.py /.gitattributes: -------------------------------------------------------------------------------- 1 | # Force LF endings 2 | * text eol=lf 3 | 4 | # Except for binaries 5 | *.tar -text 6 | *.zip -text 7 | *.gz -text 8 | *.pdf -text 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | #third_party/ 2 | #defns.mk # Only track defns.mk.template 3 | third_party/bin 4 | third_party/include 5 | third_party/src 6 | third_party/share 7 | third_party/lib 8 | 9 | # Only include defns.mk.template in repo 10 | defns.mk 11 | 12 | # Gradle stuff 13 | .gradle 14 | build/ 15 | 16 | # Ignore Gradle GUI config 17 | gradle-app.setting 18 | 19 | # # Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) 20 | !gradle-wrapper.jar 21 | 22 | # Compiled Object files 23 | *.slo 24 | *.lo 25 | *.o 26 | *.obj 27 | 28 | # Precompiled Headers 29 | *.gch 30 | *.pch 31 | 32 | # Compiled Dynamic libraries 33 | *.so 34 | *.dylib 35 | *.dll 36 | 37 | # Fortran module files 38 | *.mod 39 | 40 | # Compiled Static libraries 41 | *.lai 42 | *.la 43 | *.a 44 | *.lib 45 | 46 | # Executables 47 | *.exe 48 | *.out 49 | *.app 50 | 51 | # Java gitignore 52 | *.class 53 | 54 | # Mobile Tools for Java (J2ME) 55 | .mtj.tmp/ 56 | 57 | # Package Files # 58 | *.jar 59 | *.war 60 | *.ear 61 | 62 | # virtual machine crash logs, see 63 | http://www.java.com/en/download/help/error_hotspot.xml 64 | hs_err_pid* 65 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014, 2015 Carnegie Mellon University 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, 8 | this list of conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in the 12 | documentation and/or other materials provided with the distribution. 13 | 14 | 3. Neither the name of the copyright holder nor the names of its contributors 15 | may be used to endorse or promote products derived from this software 16 | without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 22 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 | POSSIBILITY OF SUCH DAMAGE. 29 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Assuming this Makefile lives in project root directory 2 | PROJECT := $(shell readlink $(dir $(lastword $(MAKEFILE_LIST))) -f) 3 | 4 | PETUUM_ROOT = $(PROJECT) 5 | 6 | include $(PROJECT)/defns.mk 7 | 8 | # defined in defns.mk 9 | SRC = $(PETUUM_SRC) 10 | LIB = $(PETUUM_LIB) 11 | THIRD_PARTY = $(PETUUM_THIRD_PARTY) 12 | THIRD_PARTY_SRC = $(PETUUM_THIRD_PARTY_SRC) 13 | THIRD_PARTY_LIB = $(PETUUM_THIRD_PARTY_LIB) 14 | THIRD_PARTY_INCLUDE = $(PETUUM_THIRD_PARTY_INCLUDE) 15 | THIRD_PARTY_BIN = $(PETUUM_THIRD_PARTY_BIN) 16 | 17 | BIN = $(PROJECT)/bin 18 | TESTS = $(PROJECT)/tests 19 | TESTS_BIN = $(BIN)/tests 20 | 21 | NEED_MKDIR = $(BIN) \ 22 | $(LIB) 23 | 24 | CXX = $(PETUUM_CXX) 25 | CXXFLAGS = $(PETUUM_CXXFLAGS) 26 | CXXFLAGS += -DPETUUM_MAX_NUM_CLIENTS=128 27 | INCFLAGS = $(PETUUM_INCFLAGS) 28 | LDFLAGS = $(PETUUM_LDFLAGS) 29 | 30 | all: path \ 31 | ps_lib \ 32 | ml_lib 33 | 34 | path: $(NEED_MKDIR) 35 | 36 | $(NEED_MKDIR): 37 | mkdir -p $@ 38 | 39 | clean: 40 | rm -rf $(BIN) $(LIB) $(PS_OBJ) $(ML_OBJ) $(PS_SN_OBJ) $(PS_COMMON_OBJ) 41 | 42 | .PHONY: all path clean 43 | 44 | include $(SRC)/petuum.mk 45 | -------------------------------------------------------------------------------- /app/NMF/script/kill.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os, sys 4 | 5 | if len(sys.argv) != 2: 6 | print "usage: %s " % sys.argv[0] 7 | sys.exit(1) 8 | 9 | host_file = sys.argv[1] 10 | prog_name = "nmf_main" 11 | 12 | # Get host IPs 13 | with open(host_file, "r") as f: 14 | hostlines = f.read().splitlines() 15 | host_ips = [line.split()[1] for line in hostlines] 16 | 17 | ssh_cmd = ( 18 | "ssh " 19 | "-o StrictHostKeyChecking=no " 20 | "-o UserKnownHostsFile=/dev/null " 21 | "-o LogLevel=quiet " 22 | ) 23 | 24 | for ip in host_ips: 25 | cmd = ssh_cmd + ip + " killall -q " + prog_name 26 | os.system(cmd) 27 | print "Done killing" -------------------------------------------------------------------------------- /app/NMF/script/launch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | from os.path import dirname, join 5 | import time 6 | 7 | hostfile_name = "machinefiles/localserver" 8 | 9 | app_dir = dirname(dirname(os.path.realpath(__file__))) 10 | proj_dir = dirname(dirname(app_dir)) 11 | 12 | hostfile = join(proj_dir, hostfile_name) 13 | 14 | ssh_cmd = ( 15 | "ssh " 16 | "-o StrictHostKeyChecking=no " 17 | "-o UserKnownHostsFile=/dev/null " 18 | ) 19 | 20 | # Get host IPs 21 | with open(hostfile, "r") as f: 22 | hostlines = f.read().splitlines() 23 | host_ips = [line.split()[1] for line in hostlines] 24 | 25 | for client_id, ip in enumerate(host_ips): 26 | cmd = ssh_cmd + ip + " " 27 | cmd += "\'python " + join(app_dir, "script/run_local.py") 28 | cmd += " %d %s\'" % (client_id, hostfile) 29 | cmd += " &" 30 | print cmd 31 | os.system(cmd) 32 | 33 | if client_id == 0: 34 | print "Waiting for first client to set up" 35 | time.sleep(2) -------------------------------------------------------------------------------- /app/NMF/script/launch_on_yarn.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | from os.path import dirname 5 | from os.path import join 6 | import time 7 | 8 | app_dir = dirname(dirname(os.path.realpath(__file__))) 9 | proj_dir = dirname(dirname(app_dir)) 10 | 11 | params = { 12 | "jar": join(proj_dir, "src/yarn/build/libs/yarn-0.5.jar") 13 | , "launch_script_path": join(app_dir, "script", "run_local.py") 14 | , "container_memory": 500 15 | , "container_vcores": 1 16 | , "master_memory": 350 17 | , "priority": 10 18 | , "num_nodes": 1 19 | } 20 | 21 | cmd = "hadoop jar " 22 | cmd += join(proj_dir, "src/yarn/build/libs/yarn-0.5.jar") + " " 23 | cmd += "cn.edu.buaa.act.petuumOnYarn.Client" 24 | cmd += "".join([" --%s %s" % (k,v) for k,v in params.items()]) 25 | print cmd 26 | os.system(cmd) -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/Array: -------------------------------------------------------------------------------- 1 | #ifndef EIGEN_ARRAY_MODULE_H 2 | #define EIGEN_ARRAY_MODULE_H 3 | 4 | // include Core first to handle Eigen2 support macros 5 | #include "Core" 6 | 7 | #ifndef EIGEN2_SUPPORT 8 | #error The Eigen/Array header does no longer exist in Eigen3. All that functionality has moved to Eigen/Core. 9 | #endif 10 | 11 | #endif // EIGEN_ARRAY_MODULE_H 12 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | include(RegexUtils) 2 | test_escape_string_as_regex() 3 | 4 | file(GLOB Eigen_directory_files "*") 5 | 6 | escape_string_as_regex(ESCAPED_CMAKE_CURRENT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}") 7 | 8 | foreach(f ${Eigen_directory_files}) 9 | if(NOT f MATCHES "\\.txt" AND NOT f MATCHES "${ESCAPED_CMAKE_CURRENT_SOURCE_DIR}/[.].+" AND NOT f MATCHES "${ESCAPED_CMAKE_CURRENT_SOURCE_DIR}/src") 10 | list(APPEND Eigen_directory_files_to_install ${f}) 11 | endif() 12 | endforeach(f ${Eigen_directory_files}) 13 | 14 | install(FILES 15 | ${Eigen_directory_files_to_install} 16 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen COMPONENT Devel 17 | ) 18 | 19 | add_subdirectory(src) 20 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/Cholesky: -------------------------------------------------------------------------------- 1 | #ifndef EIGEN_CHOLESKY_MODULE_H 2 | #define EIGEN_CHOLESKY_MODULE_H 3 | 4 | #include "Core" 5 | 6 | #include "src/Core/util/DisableStupidWarnings.h" 7 | 8 | /** \defgroup Cholesky_Module Cholesky module 9 | * 10 | * 11 | * 12 | * This module provides two variants of the Cholesky decomposition for selfadjoint (hermitian) matrices. 13 | * Those decompositions are accessible via the following MatrixBase methods: 14 | * - MatrixBase::llt(), 15 | * - MatrixBase::ldlt() 16 | * 17 | * \code 18 | * #include 19 | * \endcode 20 | */ 21 | 22 | #include "src/misc/Solve.h" 23 | #include "src/Cholesky/LLT.h" 24 | #include "src/Cholesky/LDLT.h" 25 | #ifdef EIGEN_USE_LAPACKE 26 | #include "src/Cholesky/LLT_MKL.h" 27 | #endif 28 | 29 | #include "src/Core/util/ReenableStupidWarnings.h" 30 | 31 | #endif // EIGEN_CHOLESKY_MODULE_H 32 | /* vim: set filetype=cpp et sw=2 ts=2 ai: */ 33 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/Dense: -------------------------------------------------------------------------------- 1 | #include "Core" 2 | #include "LU" 3 | #include "Cholesky" 4 | #include "QR" 5 | #include "SVD" 6 | #include "Geometry" 7 | #include "Eigenvalues" 8 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/Eigen: -------------------------------------------------------------------------------- 1 | #include "Dense" 2 | //#include "Sparse" 3 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/Eigenvalues: -------------------------------------------------------------------------------- 1 | #ifndef EIGEN_EIGENVALUES_MODULE_H 2 | #define EIGEN_EIGENVALUES_MODULE_H 3 | 4 | #include "Core" 5 | 6 | #include "src/Core/util/DisableStupidWarnings.h" 7 | 8 | #include "Cholesky" 9 | #include "Jacobi" 10 | #include "Householder" 11 | #include "LU" 12 | #include "Geometry" 13 | 14 | /** \defgroup Eigenvalues_Module Eigenvalues module 15 | * 16 | * 17 | * 18 | * This module mainly provides various eigenvalue solvers. 19 | * This module also provides some MatrixBase methods, including: 20 | * - MatrixBase::eigenvalues(), 21 | * - MatrixBase::operatorNorm() 22 | * 23 | * \code 24 | * #include 25 | * \endcode 26 | */ 27 | 28 | #include "src/Eigenvalues/Tridiagonalization.h" 29 | #include "src/Eigenvalues/RealSchur.h" 30 | #include "src/Eigenvalues/EigenSolver.h" 31 | #include "src/Eigenvalues/SelfAdjointEigenSolver.h" 32 | #include "src/Eigenvalues/GeneralizedSelfAdjointEigenSolver.h" 33 | #include "src/Eigenvalues/HessenbergDecomposition.h" 34 | #include "src/Eigenvalues/ComplexSchur.h" 35 | #include "src/Eigenvalues/ComplexEigenSolver.h" 36 | #include "src/Eigenvalues/RealQZ.h" 37 | #include "src/Eigenvalues/GeneralizedEigenSolver.h" 38 | #include "src/Eigenvalues/MatrixBaseEigenvalues.h" 39 | #ifdef EIGEN_USE_LAPACKE 40 | #include "src/Eigenvalues/RealSchur_MKL.h" 41 | #include "src/Eigenvalues/ComplexSchur_MKL.h" 42 | #include "src/Eigenvalues/SelfAdjointEigenSolver_MKL.h" 43 | #endif 44 | 45 | #include "src/Core/util/ReenableStupidWarnings.h" 46 | 47 | #endif // EIGEN_EIGENVALUES_MODULE_H 48 | /* vim: set filetype=cpp et sw=2 ts=2 ai: */ 49 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/Householder: -------------------------------------------------------------------------------- 1 | #ifndef EIGEN_HOUSEHOLDER_MODULE_H 2 | #define EIGEN_HOUSEHOLDER_MODULE_H 3 | 4 | #include "Core" 5 | 6 | #include "src/Core/util/DisableStupidWarnings.h" 7 | 8 | /** \defgroup Householder_Module Householder module 9 | * This module provides Householder transformations. 10 | * 11 | * \code 12 | * #include 13 | * \endcode 14 | */ 15 | 16 | #include "src/Householder/Householder.h" 17 | #include "src/Householder/HouseholderSequence.h" 18 | #include "src/Householder/BlockHouseholder.h" 19 | 20 | #include "src/Core/util/ReenableStupidWarnings.h" 21 | 22 | #endif // EIGEN_HOUSEHOLDER_MODULE_H 23 | /* vim: set filetype=cpp et sw=2 ts=2 ai: */ 24 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/Jacobi: -------------------------------------------------------------------------------- 1 | #ifndef EIGEN_JACOBI_MODULE_H 2 | #define EIGEN_JACOBI_MODULE_H 3 | 4 | #include "Core" 5 | 6 | #include "src/Core/util/DisableStupidWarnings.h" 7 | 8 | /** \defgroup Jacobi_Module Jacobi module 9 | * This module provides Jacobi and Givens rotations. 10 | * 11 | * \code 12 | * #include 13 | * \endcode 14 | * 15 | * In addition to listed classes, it defines the two following MatrixBase methods to apply a Jacobi or Givens rotation: 16 | * - MatrixBase::applyOnTheLeft() 17 | * - MatrixBase::applyOnTheRight(). 18 | */ 19 | 20 | #include "src/Jacobi/Jacobi.h" 21 | 22 | #include "src/Core/util/ReenableStupidWarnings.h" 23 | 24 | #endif // EIGEN_JACOBI_MODULE_H 25 | /* vim: set filetype=cpp et sw=2 ts=2 ai: */ 26 | 27 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/LU: -------------------------------------------------------------------------------- 1 | #ifndef EIGEN_LU_MODULE_H 2 | #define EIGEN_LU_MODULE_H 3 | 4 | #include "Core" 5 | 6 | #include "src/Core/util/DisableStupidWarnings.h" 7 | 8 | /** \defgroup LU_Module LU module 9 | * This module includes %LU decomposition and related notions such as matrix inversion and determinant. 10 | * This module defines the following MatrixBase methods: 11 | * - MatrixBase::inverse() 12 | * - MatrixBase::determinant() 13 | * 14 | * \code 15 | * #include 16 | * \endcode 17 | */ 18 | 19 | #include "src/misc/Solve.h" 20 | #include "src/misc/Kernel.h" 21 | #include "src/misc/Image.h" 22 | #include "src/LU/FullPivLU.h" 23 | #include "src/LU/PartialPivLU.h" 24 | #ifdef EIGEN_USE_LAPACKE 25 | #include "src/LU/PartialPivLU_MKL.h" 26 | #endif 27 | #include "src/LU/Determinant.h" 28 | #include "src/LU/Inverse.h" 29 | 30 | #if defined EIGEN_VECTORIZE_SSE 31 | #include "src/LU/arch/Inverse_SSE.h" 32 | #endif 33 | 34 | #ifdef EIGEN2_SUPPORT 35 | #include "src/Eigen2Support/LU.h" 36 | #endif 37 | 38 | #include "src/Core/util/ReenableStupidWarnings.h" 39 | 40 | #endif // EIGEN_LU_MODULE_H 41 | /* vim: set filetype=cpp et sw=2 ts=2 ai: */ 42 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/LeastSquares: -------------------------------------------------------------------------------- 1 | #ifndef EIGEN_REGRESSION_MODULE_H 2 | #define EIGEN_REGRESSION_MODULE_H 3 | 4 | #ifndef EIGEN2_SUPPORT 5 | #error LeastSquares is only available in Eigen2 support mode (define EIGEN2_SUPPORT) 6 | #endif 7 | 8 | // exclude from normal eigen3-only documentation 9 | #ifdef EIGEN2_SUPPORT 10 | 11 | #include "Core" 12 | 13 | #include "src/Core/util/DisableStupidWarnings.h" 14 | 15 | #include "Eigenvalues" 16 | #include "Geometry" 17 | 18 | /** \defgroup LeastSquares_Module LeastSquares module 19 | * This module provides linear regression and related features. 20 | * 21 | * \code 22 | * #include 23 | * \endcode 24 | */ 25 | 26 | #include "src/Eigen2Support/LeastSquares.h" 27 | 28 | #include "src/Core/util/ReenableStupidWarnings.h" 29 | 30 | #endif // EIGEN2_SUPPORT 31 | 32 | #endif // EIGEN_REGRESSION_MODULE_H 33 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/MetisSupport: -------------------------------------------------------------------------------- 1 | #ifndef EIGEN_METISSUPPORT_MODULE_H 2 | #define EIGEN_METISSUPPORT_MODULE_H 3 | 4 | #include "SparseCore" 5 | 6 | #include "src/Core/util/DisableStupidWarnings.h" 7 | 8 | extern "C" { 9 | #include 10 | } 11 | 12 | 13 | /** \ingroup Support_modules 14 | * \defgroup MetisSupport_Module MetisSupport module 15 | * 16 | * \code 17 | * #include 18 | * \endcode 19 | * This module defines an interface to the METIS reordering package (http://glaros.dtc.umn.edu/gkhome/views/metis). 20 | * It can be used just as any other built-in method as explained in \link OrderingMethods_Module here. \endlink 21 | */ 22 | 23 | 24 | #include "src/MetisSupport/MetisSupport.h" 25 | 26 | #include "src/Core/util/ReenableStupidWarnings.h" 27 | 28 | #endif // EIGEN_METISSUPPORT_MODULE_H 29 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/PaStiXSupport: -------------------------------------------------------------------------------- 1 | #ifndef EIGEN_PASTIXSUPPORT_MODULE_H 2 | #define EIGEN_PASTIXSUPPORT_MODULE_H 3 | 4 | #include "SparseCore" 5 | 6 | #include "src/Core/util/DisableStupidWarnings.h" 7 | 8 | #include 9 | extern "C" { 10 | #include 11 | #include 12 | } 13 | 14 | #ifdef complex 15 | #undef complex 16 | #endif 17 | 18 | /** \ingroup Support_modules 19 | * \defgroup PaStiXSupport_Module PaStiXSupport module 20 | * 21 | * This module provides an interface to the PaSTiX library. 22 | * PaSTiX is a general \b supernodal, \b parallel and \b opensource sparse solver. 23 | * It provides the two following main factorization classes: 24 | * - class PastixLLT : a supernodal, parallel LLt Cholesky factorization. 25 | * - class PastixLDLT: a supernodal, parallel LDLt Cholesky factorization. 26 | * - class PastixLU : a supernodal, parallel LU factorization (optimized for a symmetric pattern). 27 | * 28 | * \code 29 | * #include 30 | * \endcode 31 | * 32 | * In order to use this module, the PaSTiX headers must be accessible from the include paths, and your binary must be linked to the PaSTiX library and its dependencies. 33 | * The dependencies depend on how PaSTiX has been compiled. 34 | * For a cmake based project, you can use our FindPaSTiX.cmake module to help you in this task. 35 | * 36 | */ 37 | 38 | #include "src/misc/Solve.h" 39 | #include "src/misc/SparseSolve.h" 40 | 41 | #include "src/PaStiXSupport/PaStiXSupport.h" 42 | 43 | 44 | #include "src/Core/util/ReenableStupidWarnings.h" 45 | 46 | #endif // EIGEN_PASTIXSUPPORT_MODULE_H 47 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/PardisoSupport: -------------------------------------------------------------------------------- 1 | #ifndef EIGEN_PARDISOSUPPORT_MODULE_H 2 | #define EIGEN_PARDISOSUPPORT_MODULE_H 3 | 4 | #include "SparseCore" 5 | 6 | #include "src/Core/util/DisableStupidWarnings.h" 7 | 8 | #include 9 | 10 | #include 11 | 12 | /** \ingroup Support_modules 13 | * \defgroup PardisoSupport_Module PardisoSupport module 14 | * 15 | * This module brings support for the Intel(R) MKL PARDISO direct sparse solvers. 16 | * 17 | * \code 18 | * #include 19 | * \endcode 20 | * 21 | * In order to use this module, the MKL headers must be accessible from the include paths, and your binary must be linked to the MKL library and its dependencies. 22 | * See this \ref TopicUsingIntelMKL "page" for more information on MKL-Eigen integration. 23 | * 24 | */ 25 | 26 | #include "src/PardisoSupport/PardisoSupport.h" 27 | 28 | #include "src/Core/util/ReenableStupidWarnings.h" 29 | 30 | #endif // EIGEN_PARDISOSUPPORT_MODULE_H 31 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/QR: -------------------------------------------------------------------------------- 1 | #ifndef EIGEN_QR_MODULE_H 2 | #define EIGEN_QR_MODULE_H 3 | 4 | #include "Core" 5 | 6 | #include "src/Core/util/DisableStupidWarnings.h" 7 | 8 | #include "Cholesky" 9 | #include "Jacobi" 10 | #include "Householder" 11 | 12 | /** \defgroup QR_Module QR module 13 | * 14 | * 15 | * 16 | * This module provides various QR decompositions 17 | * This module also provides some MatrixBase methods, including: 18 | * - MatrixBase::qr(), 19 | * 20 | * \code 21 | * #include 22 | * \endcode 23 | */ 24 | 25 | #include "src/misc/Solve.h" 26 | #include "src/QR/HouseholderQR.h" 27 | #include "src/QR/FullPivHouseholderQR.h" 28 | #include "src/QR/ColPivHouseholderQR.h" 29 | #ifdef EIGEN_USE_LAPACKE 30 | #include "src/QR/HouseholderQR_MKL.h" 31 | #include "src/QR/ColPivHouseholderQR_MKL.h" 32 | #endif 33 | 34 | #ifdef EIGEN2_SUPPORT 35 | #include "src/Eigen2Support/QR.h" 36 | #endif 37 | 38 | #include "src/Core/util/ReenableStupidWarnings.h" 39 | 40 | #ifdef EIGEN2_SUPPORT 41 | #include "Eigenvalues" 42 | #endif 43 | 44 | #endif // EIGEN_QR_MODULE_H 45 | /* vim: set filetype=cpp et sw=2 ts=2 ai: */ 46 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/QtAlignedMalloc: -------------------------------------------------------------------------------- 1 | 2 | #ifndef EIGEN_QTMALLOC_MODULE_H 3 | #define EIGEN_QTMALLOC_MODULE_H 4 | 5 | #include "Core" 6 | 7 | #if (!EIGEN_MALLOC_ALREADY_ALIGNED) 8 | 9 | #include "src/Core/util/DisableStupidWarnings.h" 10 | 11 | void *qMalloc(size_t size) 12 | { 13 | return Eigen::internal::aligned_malloc(size); 14 | } 15 | 16 | void qFree(void *ptr) 17 | { 18 | Eigen::internal::aligned_free(ptr); 19 | } 20 | 21 | void *qRealloc(void *ptr, size_t size) 22 | { 23 | void* newPtr = Eigen::internal::aligned_malloc(size); 24 | memcpy(newPtr, ptr, size); 25 | Eigen::internal::aligned_free(ptr); 26 | return newPtr; 27 | } 28 | 29 | #include "src/Core/util/ReenableStupidWarnings.h" 30 | 31 | #endif 32 | 33 | #endif // EIGEN_QTMALLOC_MODULE_H 34 | /* vim: set filetype=cpp et sw=2 ts=2 ai: */ 35 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/SPQRSupport: -------------------------------------------------------------------------------- 1 | #ifndef EIGEN_SPQRSUPPORT_MODULE_H 2 | #define EIGEN_SPQRSUPPORT_MODULE_H 3 | 4 | #include "SparseCore" 5 | 6 | #include "src/Core/util/DisableStupidWarnings.h" 7 | 8 | #include "SuiteSparseQR.hpp" 9 | 10 | /** \ingroup Support_modules 11 | * \defgroup SPQRSupport_Module SuiteSparseQR module 12 | * 13 | * This module provides an interface to the SPQR library, which is part of the suitesparse package. 14 | * 15 | * \code 16 | * #include 17 | * \endcode 18 | * 19 | * In order to use this module, the SPQR headers must be accessible from the include paths, and your binary must be linked to the SPQR library and its dependencies (Cholmod, AMD, COLAMD,...). 20 | * For a cmake based project, you can use our FindSPQR.cmake and FindCholmod.Cmake modules 21 | * 22 | */ 23 | 24 | #include "src/misc/Solve.h" 25 | #include "src/misc/SparseSolve.h" 26 | #include "src/CholmodSupport/CholmodSupport.h" 27 | #include "src/SPQRSupport/SuiteSparseQRSupport.h" 28 | 29 | #endif 30 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/SVD: -------------------------------------------------------------------------------- 1 | #ifndef EIGEN_SVD_MODULE_H 2 | #define EIGEN_SVD_MODULE_H 3 | 4 | #include "QR" 5 | #include "Householder" 6 | #include "Jacobi" 7 | 8 | #include "src/Core/util/DisableStupidWarnings.h" 9 | 10 | /** \defgroup SVD_Module SVD module 11 | * 12 | * 13 | * 14 | * This module provides SVD decomposition for matrices (both real and complex). 15 | * This decomposition is accessible via the following MatrixBase method: 16 | * - MatrixBase::jacobiSvd() 17 | * 18 | * \code 19 | * #include 20 | * \endcode 21 | */ 22 | 23 | #include "src/misc/Solve.h" 24 | #include "src/SVD/JacobiSVD.h" 25 | #if defined(EIGEN_USE_LAPACKE) && !defined(EIGEN_USE_LAPACKE_STRICT) 26 | #include "src/SVD/JacobiSVD_MKL.h" 27 | #endif 28 | #include "src/SVD/UpperBidiagonalization.h" 29 | 30 | #ifdef EIGEN2_SUPPORT 31 | #include "src/Eigen2Support/SVD.h" 32 | #endif 33 | 34 | #include "src/Core/util/ReenableStupidWarnings.h" 35 | 36 | #endif // EIGEN_SVD_MODULE_H 37 | /* vim: set filetype=cpp et sw=2 ts=2 ai: */ 38 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/Sparse: -------------------------------------------------------------------------------- 1 | #ifndef EIGEN_SPARSE_MODULE_H 2 | #define EIGEN_SPARSE_MODULE_H 3 | 4 | /** \defgroup Sparse_Module Sparse meta-module 5 | * 6 | * Meta-module including all related modules: 7 | * - \ref SparseCore_Module 8 | * - \ref OrderingMethods_Module 9 | * - \ref SparseCholesky_Module 10 | * - \ref SparseLU_Module 11 | * - \ref SparseQR_Module 12 | * - \ref IterativeLinearSolvers_Module 13 | * 14 | * \code 15 | * #include 16 | * \endcode 17 | */ 18 | 19 | #include "SparseCore" 20 | #include "OrderingMethods" 21 | #include "SparseCholesky" 22 | #include "SparseLU" 23 | #include "SparseQR" 24 | #include "IterativeLinearSolvers" 25 | 26 | #endif // EIGEN_SPARSE_MODULE_H 27 | 28 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/SparseCholesky: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2008-2013 Gael Guennebaud 5 | // 6 | // This Source Code Form is subject to the terms of the Mozilla 7 | // Public License v. 2.0. If a copy of the MPL was not distributed 8 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | 10 | #ifndef EIGEN_SPARSECHOLESKY_MODULE_H 11 | #define EIGEN_SPARSECHOLESKY_MODULE_H 12 | 13 | #include "SparseCore" 14 | #include "OrderingMethods" 15 | 16 | #include "src/Core/util/DisableStupidWarnings.h" 17 | 18 | /** 19 | * \defgroup SparseCholesky_Module SparseCholesky module 20 | * 21 | * This module currently provides two variants of the direct sparse Cholesky decomposition for selfadjoint (hermitian) matrices. 22 | * Those decompositions are accessible via the following classes: 23 | * - SimplicialLLt, 24 | * - SimplicialLDLt 25 | * 26 | * Such problems can also be solved using the ConjugateGradient solver from the IterativeLinearSolvers module. 27 | * 28 | * \code 29 | * #include 30 | * \endcode 31 | */ 32 | 33 | #ifdef EIGEN_MPL2_ONLY 34 | #error The SparseCholesky module has nothing to offer in MPL2 only mode 35 | #endif 36 | 37 | #include "src/misc/Solve.h" 38 | #include "src/misc/SparseSolve.h" 39 | #include "src/SparseCholesky/SimplicialCholesky.h" 40 | 41 | #ifndef EIGEN_MPL2_ONLY 42 | #include "src/SparseCholesky/SimplicialCholesky_impl.h" 43 | #endif 44 | 45 | #include "src/Core/util/ReenableStupidWarnings.h" 46 | 47 | #endif // EIGEN_SPARSECHOLESKY_MODULE_H 48 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/SparseQR: -------------------------------------------------------------------------------- 1 | #ifndef EIGEN_SPARSEQR_MODULE_H 2 | #define EIGEN_SPARSEQR_MODULE_H 3 | 4 | #include "SparseCore" 5 | #include "OrderingMethods" 6 | #include "src/Core/util/DisableStupidWarnings.h" 7 | 8 | /** \defgroup SparseQR_Module SparseQR module 9 | * \brief Provides QR decomposition for sparse matrices 10 | * 11 | * This module provides a simplicial version of the left-looking Sparse QR decomposition. 12 | * The columns of the input matrix should be reordered to limit the fill-in during the 13 | * decomposition. Built-in methods (COLAMD, AMD) or external methods (METIS) can be used to this end. 14 | * See the \link OrderingMethods_Module OrderingMethods\endlink module for the list 15 | * of built-in and external ordering methods. 16 | * 17 | * \code 18 | * #include 19 | * \endcode 20 | * 21 | * 22 | */ 23 | 24 | #include "src/misc/Solve.h" 25 | #include "src/misc/SparseSolve.h" 26 | 27 | #include "OrderingMethods" 28 | #include "src/SparseCore/SparseColEtree.h" 29 | #include "src/SparseQR/SparseQR.h" 30 | 31 | #include "src/Core/util/ReenableStupidWarnings.h" 32 | 33 | #endif 34 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/StdDeque: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2009 Gael Guennebaud 5 | // Copyright (C) 2009 Hauke Heibel 6 | // 7 | // This Source Code Form is subject to the terms of the Mozilla 8 | // Public License v. 2.0. If a copy of the MPL was not distributed 9 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 10 | 11 | #ifndef EIGEN_STDDEQUE_MODULE_H 12 | #define EIGEN_STDDEQUE_MODULE_H 13 | 14 | #include "Core" 15 | #include 16 | 17 | #if (defined(_MSC_VER) && defined(_WIN64)) /* MSVC auto aligns in 64 bit builds */ 18 | 19 | #define EIGEN_DEFINE_STL_DEQUE_SPECIALIZATION(...) 20 | 21 | #else 22 | 23 | #include "src/StlSupport/StdDeque.h" 24 | 25 | #endif 26 | 27 | #endif // EIGEN_STDDEQUE_MODULE_H 28 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/StdList: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2009 Hauke Heibel 5 | // 6 | // This Source Code Form is subject to the terms of the Mozilla 7 | // Public License v. 2.0. If a copy of the MPL was not distributed 8 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | 10 | #ifndef EIGEN_STDLIST_MODULE_H 11 | #define EIGEN_STDLIST_MODULE_H 12 | 13 | #include "Core" 14 | #include 15 | 16 | #if (defined(_MSC_VER) && defined(_WIN64)) /* MSVC auto aligns in 64 bit builds */ 17 | 18 | #define EIGEN_DEFINE_STL_LIST_SPECIALIZATION(...) 19 | 20 | #else 21 | 22 | #include "src/StlSupport/StdList.h" 23 | 24 | #endif 25 | 26 | #endif // EIGEN_STDLIST_MODULE_H 27 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/StdVector: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2009 Gael Guennebaud 5 | // Copyright (C) 2009 Hauke Heibel 6 | // 7 | // This Source Code Form is subject to the terms of the Mozilla 8 | // Public License v. 2.0. If a copy of the MPL was not distributed 9 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 10 | 11 | #ifndef EIGEN_STDVECTOR_MODULE_H 12 | #define EIGEN_STDVECTOR_MODULE_H 13 | 14 | #include "Core" 15 | #include 16 | 17 | #if (defined(_MSC_VER) && defined(_WIN64)) /* MSVC auto aligns in 64 bit builds */ 18 | 19 | #define EIGEN_DEFINE_STL_VECTOR_SPECIALIZATION(...) 20 | 21 | #else 22 | 23 | #include "src/StlSupport/StdVector.h" 24 | 25 | #endif 26 | 27 | #endif // EIGEN_STDVECTOR_MODULE_H 28 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/UmfPackSupport: -------------------------------------------------------------------------------- 1 | #ifndef EIGEN_UMFPACKSUPPORT_MODULE_H 2 | #define EIGEN_UMFPACKSUPPORT_MODULE_H 3 | 4 | #include "SparseCore" 5 | 6 | #include "src/Core/util/DisableStupidWarnings.h" 7 | 8 | extern "C" { 9 | #include 10 | } 11 | 12 | /** \ingroup Support_modules 13 | * \defgroup UmfPackSupport_Module UmfPackSupport module 14 | * 15 | * This module provides an interface to the UmfPack library which is part of the suitesparse package. 16 | * It provides the following factorization class: 17 | * - class UmfPackLU: a multifrontal sequential LU factorization. 18 | * 19 | * \code 20 | * #include 21 | * \endcode 22 | * 23 | * In order to use this module, the umfpack headers must be accessible from the include paths, and your binary must be linked to the umfpack library and its dependencies. 24 | * The dependencies depend on how umfpack has been compiled. 25 | * For a cmake based project, you can use our FindUmfPack.cmake module to help you in this task. 26 | * 27 | */ 28 | 29 | #include "src/misc/Solve.h" 30 | #include "src/misc/SparseSolve.h" 31 | 32 | #include "src/UmfPackSupport/UmfPackSupport.h" 33 | 34 | #include "src/Core/util/ReenableStupidWarnings.h" 35 | 36 | #endif // EIGEN_UMFPACKSUPPORT_MODULE_H 37 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB Eigen_src_subdirectories "*") 2 | escape_string_as_regex(ESCAPED_CMAKE_CURRENT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}") 3 | foreach(f ${Eigen_src_subdirectories}) 4 | if(NOT f MATCHES "\\.txt" AND NOT f MATCHES "${ESCAPED_CMAKE_CURRENT_SOURCE_DIR}/[.].+" ) 5 | add_subdirectory(${f}) 6 | endif() 7 | endforeach() 8 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/Cholesky/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_Cholesky_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_Cholesky_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Cholesky COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/CholmodSupport/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_CholmodSupport_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_CholmodSupport_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/CholmodSupport COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/Core/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_Core_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_Core_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Core COMPONENT Devel 6 | ) 7 | 8 | ADD_SUBDIRECTORY(products) 9 | ADD_SUBDIRECTORY(util) 10 | ADD_SUBDIRECTORY(arch) 11 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/Core/arch/AltiVec/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_Core_arch_AltiVec_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_Core_arch_AltiVec_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Core/arch/AltiVec COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/Core/arch/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | ADD_SUBDIRECTORY(SSE) 2 | ADD_SUBDIRECTORY(AltiVec) 3 | ADD_SUBDIRECTORY(NEON) 4 | ADD_SUBDIRECTORY(Default) 5 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/Core/arch/Default/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_Core_arch_Default_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_Core_arch_Default_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Core/arch/Default COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/Core/arch/NEON/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_Core_arch_NEON_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_Core_arch_NEON_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Core/arch/NEON COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/Core/arch/SSE/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_Core_arch_SSE_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_Core_arch_SSE_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Core/arch/SSE COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/Core/products/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_Core_Product_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_Core_Product_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Core/products COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/Core/util/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_Core_util_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_Core_util_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Core/util COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/Core/util/NonMPL2.h: -------------------------------------------------------------------------------- 1 | #ifdef EIGEN_MPL2_ONLY 2 | #error Including non-MPL2 code in EIGEN_MPL2_ONLY mode 3 | #endif 4 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/Core/util/ReenableStupidWarnings.h: -------------------------------------------------------------------------------- 1 | #ifdef EIGEN_WARNINGS_DISABLED 2 | #undef EIGEN_WARNINGS_DISABLED 3 | 4 | #ifndef EIGEN_PERMANENTLY_DISABLE_STUPID_WARNINGS 5 | #ifdef _MSC_VER 6 | #pragma warning( pop ) 7 | #elif defined __INTEL_COMPILER 8 | #pragma warning pop 9 | #elif defined __clang__ 10 | #pragma clang diagnostic pop 11 | #endif 12 | #endif 13 | 14 | #endif // EIGEN_WARNINGS_DISABLED 15 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/Eigen2Support/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_Eigen2Support_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_Eigen2Support_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Eigen2Support COMPONENT Devel 6 | ) 7 | 8 | ADD_SUBDIRECTORY(Geometry) -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/Eigen2Support/Geometry/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_Eigen2Support_Geometry_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_Eigen2Support_Geometry_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Eigen2Support/Geometry 6 | ) 7 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/Eigen2Support/Macros.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2011 Benoit Jacob 5 | // 6 | // This Source Code Form is subject to the terms of the Mozilla 7 | // Public License v. 2.0. If a copy of the MPL was not distributed 8 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | 10 | #ifndef EIGEN2_MACROS_H 11 | #define EIGEN2_MACROS_H 12 | 13 | #define ei_assert eigen_assert 14 | #define ei_internal_assert eigen_internal_assert 15 | 16 | #define EIGEN_ALIGN_128 EIGEN_ALIGN16 17 | 18 | #define EIGEN_ARCH_WANTS_ALIGNMENT EIGEN_ALIGN_STATICALLY 19 | 20 | #endif // EIGEN2_MACROS_H 21 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/Eigenvalues/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_EIGENVALUES_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_EIGENVALUES_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Eigenvalues COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/Geometry/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_Geometry_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_Geometry_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Geometry COMPONENT Devel 6 | ) 7 | 8 | ADD_SUBDIRECTORY(arch) 9 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/Geometry/arch/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_Geometry_arch_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_Geometry_arch_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Geometry/arch COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/Householder/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_Householder_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_Householder_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Householder COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/IterativeLinearSolvers/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_IterativeLinearSolvers_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_IterativeLinearSolvers_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/IterativeLinearSolvers COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/Jacobi/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_Jacobi_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_Jacobi_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Jacobi COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/LU/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_LU_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_LU_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/LU COMPONENT Devel 6 | ) 7 | 8 | ADD_SUBDIRECTORY(arch) 9 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/LU/arch/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_LU_arch_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_LU_arch_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/LU/arch COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/MetisSupport/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_MetisSupport_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_MetisSupport_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/MetisSupport COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/OrderingMethods/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_OrderingMethods_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_OrderingMethods_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/OrderingMethods COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/PaStiXSupport/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_PastixSupport_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_PastixSupport_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/PaStiXSupport COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/PardisoSupport/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_PardisoSupport_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_PardisoSupport_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/PardisoSupport COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/QR/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_QR_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_QR_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/QR COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/SPQRSupport/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_SPQRSupport_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_SPQRSupport_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/SPQRSupport/ COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/SVD/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_SVD_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_SVD_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/SVD COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/SparseCholesky/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_SparseCholesky_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_SparseCholesky_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/SparseCholesky COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/SparseCore/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_SparseCore_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_SparseCore_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/SparseCore COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/SparseCore/SparseFuzzy.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2008 Gael Guennebaud 5 | // 6 | // This Source Code Form is subject to the terms of the Mozilla 7 | // Public License v. 2.0. If a copy of the MPL was not distributed 8 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | 10 | #ifndef EIGEN_SPARSE_FUZZY_H 11 | #define EIGEN_SPARSE_FUZZY_H 12 | 13 | // template 14 | // template 15 | // bool SparseMatrixBase::isApprox( 16 | // const OtherDerived& other, 17 | // typename NumTraits::Real prec 18 | // ) const 19 | // { 20 | // const typename internal::nested::type nested(derived()); 21 | // const typename internal::nested::type otherNested(other.derived()); 22 | // return (nested - otherNested).cwise().abs2().sum() 23 | // <= prec * prec * (std::min)(nested.cwise().abs2().sum(), otherNested.cwise().abs2().sum()); 24 | // } 25 | 26 | #endif // EIGEN_SPARSE_FUZZY_H 27 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/SparseLU/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_SparseLU_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_SparseLU_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/SparseLU COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/SparseQR/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_SparseQR_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_SparseQR_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/SparseQR/ COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/StlSupport/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_StlSupport_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_StlSupport_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/StlSupport COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/SuperLUSupport/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_SuperLUSupport_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_SuperLUSupport_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/SuperLUSupport COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/UmfPackSupport/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_UmfPackSupport_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_UmfPackSupport_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/UmfPackSupport COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/misc/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_misc_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_misc_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/misc COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/NMF/src/util/Eigen/src/plugins/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_plugins_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_plugins_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/plugins COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/dml/Makefile: -------------------------------------------------------------------------------- 1 | DML_DIR := $(shell readlink $(dir $(lastword $(MAKEFILE_LIST))) -f) 2 | PETUUM_ROOT = $(DML_DIR)/../../ 3 | 4 | include $(PETUUM_ROOT)/defns.mk 5 | 6 | DML_SRC = $(wildcard $(DML_DIR)/src/dml/*.cpp) 7 | DML_HDR = $(wildcard $(MLR_DIR)/src/dml/*.hpp) 8 | 9 | DML_SRC += $(wildcard $(PETUUM_ROOT)/src/io/*.cpp) 10 | 11 | DML_HDR += $(wildcard $(PETUUM_ROOT)/src/io/*.hpp) 12 | 13 | DML_BIN = $(DML_DIR)/bin 14 | DML_OBJ = $(DML_SRC:.cpp=.o) 15 | 16 | PETUUM_INCFLAGS+= ${HAS_HDFS} 17 | PETUUM_CXXFLAGS+= -fopenmp 18 | 19 | all: DML 20 | 21 | DML: $(DML_BIN)/DML 22 | 23 | 24 | $(DML_BIN): 25 | mkdir -p $(DML_BIN) 26 | 27 | $(DML_BIN)/DML: $(DML_OBJ) $(PETUUM_PS_LIB) $(PETUUM_ML_LIB) $(DML_BIN) 28 | $(PETUUM_CXX) $(PETUUM_CXXFLAGS) $(PETUUM_INCFLAGS) \ 29 | $(DML_OBJ) $(PETUUM_PS_LIB) $(PETUUM_ML_LIB) $(PETUUM_LDFLAGS) -o $@ 30 | 31 | $(DML_OBJ): %.o: %.cpp $(DML_HDR) 32 | $(PETUUM_CXX) $(NDEBUG) $(PETUUM_CXXFLAGS) -Wno-unused-result $(PETUUM_INCFLAGS) -c $< -o $@ 33 | 34 | $(DML_BIN)/DML_sn: $(DML_SN_OBJ) $(PETUUM_PS_SN_LIB) $(DML_BIN) 35 | $(PETUUM_CXX) $(PETUUM_CXXFLAGS) -DPETUUM_SINGLE_NODE $(PETUUM_INCFLAGS) \ 36 | $(DML_SN_OBJ) $(PETUUM_PS_SN_LIB) $(PETUUM_LDFLAGS) -o $@ 37 | 38 | 39 | 40 | 41 | clean: 42 | rm -rf $(DML_OBJ) 43 | rm -rf $(DML_BIN) 44 | 45 | .PHONY: clean DML 46 | -------------------------------------------------------------------------------- /app/dml/datasets/dml_para.txt: -------------------------------------------------------------------------------- 1 | src_feat_dim: 780 2 | dst_feat_dim: 600 3 | lambda: 1 4 | thre: 1 5 | learn_rate: 0.001 6 | epoch: 50 7 | num_total_pts: 60000 8 | num_simi_pairs: 1000 9 | num_diff_pairs: 1000 10 | mini_batch_size: 100 11 | num_iters_evaluate: 1000 12 | num_smps_evaluate: 1000 13 | -------------------------------------------------------------------------------- /app/dml/script/kill.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os, sys 4 | 5 | if len(sys.argv) != 2: 6 | print "usage: %s " % sys.argv[0] 7 | sys.exit(1) 8 | 9 | host_file = sys.argv[1] 10 | prog_name = "DML" 11 | 12 | # Get host IPs 13 | with open(host_file, "r") as f: 14 | hostlines = f.read().splitlines() 15 | host_ips = [line.split()[1] for line in hostlines] 16 | 17 | ssh_cmd = ( 18 | "ssh " 19 | "-o StrictHostKeyChecking=no " 20 | "-o UserKnownHostsFile=/dev/null " 21 | "-o LogLevel=quiet " 22 | ) 23 | 24 | for ip in host_ips: 25 | cmd = ssh_cmd + ip + " killall -q " + prog_name 26 | os.system(cmd) 27 | print "Done killing" -------------------------------------------------------------------------------- /app/dml/script/launch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | from os.path import dirname, join 5 | import time 6 | 7 | hostfile_name = "machinefiles/localserver" 8 | 9 | app_dir = dirname(dirname(os.path.realpath(__file__))) 10 | proj_dir = dirname(dirname(app_dir)) 11 | 12 | hostfile = join(proj_dir, hostfile_name) 13 | 14 | ssh_cmd = ( 15 | "ssh " 16 | "-o StrictHostKeyChecking=no " 17 | "-o UserKnownHostsFile=/dev/null " 18 | ) 19 | 20 | # Get host IPs 21 | with open(hostfile, "r") as f: 22 | hostlines = f.read().splitlines() 23 | host_ips = [line.split()[1] for line in hostlines] 24 | 25 | for client_id, ip in enumerate(host_ips): 26 | cmd = ssh_cmd + ip + " " 27 | cmd += "\'python " + join(app_dir, "script/run_local.py") 28 | cmd += " %d %s\'" % (client_id, hostfile) 29 | cmd += " &" 30 | print cmd 31 | os.system(cmd) 32 | 33 | if client_id == 0: 34 | print "Waiting for first client to set up" 35 | time.sleep(2) 36 | -------------------------------------------------------------------------------- /app/dml/script/launch_on_yarn.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | from os.path import dirname 5 | from os.path import join 6 | import time 7 | 8 | app_dir = dirname(dirname(os.path.realpath(__file__))) 9 | proj_dir = dirname(dirname(app_dir)) 10 | 11 | params = { 12 | "jar": join(proj_dir, "src/yarn/build/libs/yarn-0.5.jar") 13 | , "launch_script_path": join(app_dir, "script", "run_local.py") 14 | , "container_memory": 500 15 | , "container_vcores": 1 16 | , "master_memory": 350 17 | , "priority": 10 18 | , "num_nodes": 1 19 | } 20 | 21 | cmd = "hadoop jar " 22 | cmd += join(proj_dir, "src/yarn/build/libs/yarn-0.5.jar") + " " 23 | cmd += "cn.edu.buaa.act.petuumOnYarn.Client" 24 | cmd += "".join([" --%s %s" % (k,v) for k,v in params.items()]) 25 | print cmd 26 | os.system(cmd) -------------------------------------------------------------------------------- /app/dml/src/dml/aux_data.hpp: -------------------------------------------------------------------------------- 1 | #ifndef AUX_DATA_H_ 2 | #define AUX_DATA_H_ 3 | 4 | struct pair { 5 | int x; 6 | int y; 7 | }; 8 | 9 | #endif -------------------------------------------------------------------------------- /app/dml/src/dml/dml_paras.hpp: -------------------------------------------------------------------------------- 1 | #ifndef APPS_DML_SRC_DML_DML_PARAS_H_ 2 | #define APPS_DML_SRC_DML_DML_PARAS_H_ 3 | 4 | struct DmlParas{ 5 | // original feature dimension 6 | int src_feat_dim; 7 | // target feature dimension 8 | int dst_feat_dim; 9 | // tradeoff parameter 10 | float lambda; 11 | // distance threshold 12 | float thre; 13 | // learning rate 14 | float learn_rate; 15 | int epoch; 16 | // number of total pts 17 | int num_total_pts; 18 | // number of similar pairs 19 | int num_simi_pairs; 20 | // num of dissimilar pairs 21 | int num_diff_pairs; 22 | // number of total evaluation pts 23 | int num_total_eval_pts; 24 | // size of mini batch 25 | int size_mb; 26 | // num iters to do evaluation 27 | int num_iters_evaluate; 28 | // num smps to evaluate 29 | int num_smps_evaluate; 30 | // uniform initialization range 31 | float unif_x; 32 | float unif_y; 33 | 34 | char data_file[512]; 35 | char simi_pair_file[512]; 36 | char diff_pair_file[512]; 37 | }; 38 | 39 | void LoadDmlParas(DmlParas * para, const char * file_dml_para); 40 | 41 | void PrintDmlParas(const DmlParas para); 42 | 43 | #endif // APPS_DML_SRC_DML_DML_PARAS_H_ 44 | -------------------------------------------------------------------------------- /app/dml/src/dml/types.hpp: -------------------------------------------------------------------------------- 1 | #ifndef TYPES_H_ 2 | #define TYPES_H_ 3 | 4 | #include 5 | 6 | typedef petuum::Table mat; 7 | typedef petuum::DenseRow row; 8 | 9 | #endif -------------------------------------------------------------------------------- /app/dml/src/dml/utils.hpp: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | #include "aux_data.hpp" 5 | 6 | void rand_init_vec_int(int * a, int dim,int max_int); 7 | int myrandom2 (int i); 8 | //get time 9 | float get_time(); 10 | 11 | float randfloat(); 12 | //z=x-y 13 | void VecSub(float * x, float * y, float * z, int dim); 14 | //y=mat*x 15 | void MatVecMul(float ** mat, float * x, float * y, int num_rows_mat, int num_cols_mat); 16 | //return ||vec||^{2} 17 | float VecSqr(float * vec,int dim); 18 | //void data sparse format 19 | void LoadSparseData(float ** data, int num_data, const char * file); 20 | //load dense data 21 | void LoadPairs(pair * pairs, int num_pairs, const char * file); 22 | -------------------------------------------------------------------------------- /app/dnn/datasets/para_imnet.txt: -------------------------------------------------------------------------------- 1 | num_layers: 4 2 | num_units_in_each_layer: 360 512 512 2001 3 | num_epochs: 20 4 | stepsize: 0.1 5 | mini_batch_size: 256 6 | num_smp_evaluate: 2000 7 | num_iters_evaluate: 10 8 | -------------------------------------------------------------------------------- /app/dnn/script/gen_data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ $# -ne 5 ]; then 3 | echo "Usage: $0 " 4 | exit 5 | fi 6 | progname=gen_data 7 | script_path=`readlink -f $0` 8 | script_dir=`dirname $script_path` 9 | project_root=`dirname $script_dir` 10 | prog_path=$project_root/bin/$progname 11 | if [ ! -d $5 ]; then 12 | mkdir $5 13 | fi 14 | $prog_path $1 $2 $3 $4 `readlink -f $5` 15 | -------------------------------------------------------------------------------- /app/dnn/script/kill.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os, sys 4 | 5 | if len(sys.argv) != 2: 6 | print "usage: %s " % sys.argv[0] 7 | sys.exit(1) 8 | 9 | host_file = sys.argv[1] 10 | prog_name = "DNN" 11 | 12 | # Get host IPs 13 | with open(host_file, "r") as f: 14 | hostlines = f.read().splitlines() 15 | host_ips = [line.split()[1] for line in hostlines] 16 | 17 | ssh_cmd = ( 18 | "ssh " 19 | "-o StrictHostKeyChecking=no " 20 | "-o UserKnownHostsFile=/dev/null " 21 | "-o LogLevel=quiet " 22 | ) 23 | 24 | for ip in host_ips: 25 | cmd = ssh_cmd + ip + " killall -q " + prog_name 26 | os.system(cmd) 27 | print "Done killing" -------------------------------------------------------------------------------- /app/dnn/script/launch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | from os.path import dirname, join 5 | import time 6 | 7 | hostfile_name = "machinefiles/localserver" 8 | 9 | app_dir = dirname(dirname(os.path.realpath(__file__))) 10 | proj_dir = dirname(dirname(app_dir)) 11 | 12 | hostfile = join(proj_dir, hostfile_name) 13 | 14 | ssh_cmd = ( 15 | "ssh " 16 | "-o StrictHostKeyChecking=no " 17 | "-o UserKnownHostsFile=/dev/null " 18 | ) 19 | 20 | # Get host IPs 21 | with open(hostfile, "r") as f: 22 | hostlines = f.read().splitlines() 23 | host_ips = [line.split()[1] for line in hostlines] 24 | 25 | for client_id, ip in enumerate(host_ips): 26 | cmd = ssh_cmd + ip + " " 27 | cmd += "\'python " + join(app_dir, "script/run_local.py") 28 | cmd += " %d %s\'" % (client_id, hostfile) 29 | cmd += " &" 30 | print cmd 31 | os.system(cmd) 32 | 33 | if client_id == 0: 34 | print "Waiting for first client to set up" 35 | time.sleep(2) 36 | -------------------------------------------------------------------------------- /app/dnn/script/launch_on_yarn.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | from os.path import dirname 5 | from os.path import join 6 | import time 7 | 8 | app_dir = dirname(dirname(os.path.realpath(__file__))) 9 | proj_dir = dirname(dirname(app_dir)) 10 | 11 | params = { 12 | "jar": join(proj_dir, "src/yarn/build/libs/yarn-0.5.jar") 13 | , "launch_script_path": join(app_dir, "script", "run_local.py") 14 | , "container_memory": 500 15 | , "container_vcores": 1 16 | , "master_memory": 350 17 | , "priority": 10 18 | , "num_nodes": 1 19 | } 20 | 21 | cmd = "hadoop jar " 22 | cmd += join(proj_dir, "src/yarn/build/libs/yarn-0.5.jar") + " " 23 | cmd += "cn.edu.buaa.act.petuumOnYarn.Client" 24 | cmd += "".join([" --%s %s" % (k,v) for k,v in params.items()]) 25 | print cmd 26 | os.system(cmd) -------------------------------------------------------------------------------- /app/dnn/script/launch_pred.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | from os.path import dirname, join 5 | import time 6 | 7 | hostfile_name = "machinefiles/localserver" 8 | 9 | app_dir = dirname(dirname(os.path.realpath(__file__))) 10 | proj_dir = dirname(dirname(app_dir)) 11 | 12 | hostfile = join(proj_dir, hostfile_name) 13 | 14 | ssh_cmd = ( 15 | "ssh " 16 | "-o StrictHostKeyChecking=no " 17 | "-o UserKnownHostsFile=/dev/null " 18 | ) 19 | 20 | # Get host IPs 21 | with open(hostfile, "r") as f: 22 | hostlines = f.read().splitlines() 23 | host_ips = [line.split()[1] for line in hostlines] 24 | 25 | for client_id, ip in enumerate(host_ips): 26 | cmd = ssh_cmd + ip + " " 27 | cmd += "\'python " + join(app_dir, "script/predict.py") 28 | cmd += " %d %s\'" % (client_id, hostfile) 29 | cmd += " &" 30 | print cmd 31 | os.system(cmd) 32 | 33 | if client_id == 0: 34 | print "Waiting for first client to set up" 35 | time.sleep(2) 36 | -------------------------------------------------------------------------------- /app/dnn_speech/Makefile: -------------------------------------------------------------------------------- 1 | DNN_DIR := $(shell readlink $(dir $(lastword $(MAKEFILE_LIST))) -f) 2 | PETUUM_ROOT = $(DNN_DIR)/../../ 3 | 4 | include $(PETUUM_ROOT)/defns.mk 5 | 6 | DNN_SRC = $(wildcard $(DNN_DIR)/src/dnn/*.cpp) 7 | DNN_HDR = $(wildcard $(DNN_DIR)/src/dnn/*.hpp) 8 | DNN_BIN = $(DNN_DIR)/bin 9 | DNN_OBJ = $(DNN_SRC:.cpp=.o) 10 | DNN_SN_OBJ = $(DNN_SRC:.cpp=_sn.o) 11 | GENDATA_SRC= $(DNN_DIR)/src/tools/gen_data.cpp 12 | 13 | #all: DNN DNN_sn GENDATA 14 | all: DNN GENDATA 15 | 16 | DNN: $(DNN_BIN)/DNN 17 | DNN_sn: $(DNN_BIN)/DNN_sn 18 | GENDATA: $(DNN_BIN)/gen_data 19 | 20 | $(DNN_BIN): 21 | mkdir -p $(DNN_BIN) 22 | 23 | $(DNN_BIN)/DNN: $(DNN_OBJ) $(PETUUM_PS_LIB) $(DNN_BIN) 24 | $(PETUUM_CXX) $(PETUUM_CXXFLAGS) $(PETUUM_INCFLAGS) \ 25 | $(DNN_OBJ) $(PETUUM_PS_LIB) $(PETUUM_LDFLAGS) -o $@ 26 | 27 | $(DNN_OBJ): %.o: %.cpp $(DNN_HDR) 28 | $(PETUUM_CXX) $(PETUUM_CXXFLAGS) -Wno-unused-result $(PETUUM_INCFLAGS) -c $< -o $@ 29 | 30 | $(DNN_BIN)/DNN_sn: $(DNN_SN_OBJ) $(PETUUM_PS_SN_LIB) $(DNN_BIN) 31 | $(PETUUM_CXX) $(PETUUM_CXXFLAGS) $(PETUUM_INCFLAGS) -DPETUUM_SINGLE_NODE \ 32 | $(DNN_SN_OBJ) $(PETUUM_PS_SN_LIB) $(PETUUM_LDFLAGS) -o $@ 33 | 34 | $(DNN_SN_OBJ): %_sn.o: %.cpp $(DNN_HDR) 35 | $(PETUUM_CXX) $(PETUUM_CXXFLAGS) -DPETUUM_SINGLE_NODE -Wno-unused-result \ 36 | $(PETUUM_INCFLAGS) -c $< -o $@ 37 | 38 | 39 | $(DNN_BIN)/gen_data: $(GENDATA_SRC) 40 | mkdir -p $(DNN_BIN) 41 | $(PETUUM_CXX) $(PETUUM_CXXFLAGS) $^ \ 42 | $(PETUUM_LDFLAGS) -o $@ 43 | 44 | clean: 45 | rm -rf $(DNN_OBJ) 46 | rm -rf $(DNN_SN_OBJ) 47 | rm -rf $(DNN_BIN) 48 | 49 | .PHONY: clean DNN DNN_sn 50 | -------------------------------------------------------------------------------- /app/dnn_speech/datasets/data_partition.txt: -------------------------------------------------------------------------------- 1 | /home/jingwei/internal/apps/dnn_speech/Train 1031950 2 | -------------------------------------------------------------------------------- /app/dnn_speech/datasets/olddata_partition.txt: -------------------------------------------------------------------------------- 1 | /home/jingwei/internal/apps/dnn/Train 1031950 2 | -------------------------------------------------------------------------------- /app/dnn_speech/datasets/para_imnet.txt: -------------------------------------------------------------------------------- 1 | num_layers: 4 2 | num_units_in_each_layer: 360 512 512 2001 3 | num_epochs: 2 4 | stepsize: 1 5 | mini_batch_size: 256 6 | num_smp_evaluate: 2000 7 | num_iters_evaluate: 10 8 | -------------------------------------------------------------------------------- /app/dnn_speech/kaldi-trunk.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sailing-pmls/bosen/06cb58902d011fbea5f9428f10ce30e621492204/app/dnn_speech/kaldi-trunk.tar.gz -------------------------------------------------------------------------------- /app/dnn_speech/machinefiles/localserver: -------------------------------------------------------------------------------- 1 | 0 127.0.0.1 9999 2 | -------------------------------------------------------------------------------- /app/dnn_speech/myrun.sh: -------------------------------------------------------------------------------- 1 | scripts/run_dnn.sh 1 5 machinefiles/localserver datasets/para_imnet.txt datasets/data_partition.txt DNN_para.txt 2 | -------------------------------------------------------------------------------- /app/dnn_speech/petuumdnn/Makefile: -------------------------------------------------------------------------------- 1 | 2 | all: 3 | EXTRA_CXXFLAGS = -Wno-sign-compare 4 | include ../kaldi.mk 5 | 6 | LDFLAGS += $(CUDA_LDFLAGS) 7 | LDLIBS += $(CUDA_LDLIBS) 8 | 9 | BINFILES = feature_extraction ali_count parse_head set_num_layers 10 | 11 | OBJFILES = 12 | 13 | # Add this dependency to force cuda-compiled.o to be rebuilt when we reconfigure. 14 | cuda-compiled.o: ../kaldi.mk 15 | 16 | 17 | TESTFILES = 18 | 19 | ADDLIBS = ../nnet2/kaldi-nnet2.a ../nnet/kaldi-nnet.a ../gmm/kaldi-gmm.a \ 20 | ../decoder/kaldi-decoder.a ../lat/kaldi-lat.a ../hmm/kaldi-hmm.a \ 21 | ../transform/kaldi-transform.a ../tree/kaldi-tree.a ../thread/kaldi-thread.a \ 22 | ../cudamatrix/kaldi-cudamatrix.a ../matrix/kaldi-matrix.a \ 23 | ../util/kaldi-util.a ../base/kaldi-base.a 24 | 25 | include ../makefiles/default_rules.mk 26 | -------------------------------------------------------------------------------- /app/dnn_speech/petuumdnn/ali_count.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | int main(int argc, char **argv) { 7 | std::vector count; 8 | std::ifstream infile(argv[1],std::ios::in); 9 | std::ofstream outfile(argv[2], std::ios::out); 10 | std::ofstream outfile2(argv[3], std::ios::out); 11 | int term, sumNum = 0, sumAli = atoi(argv[4]), input_dim = atoi(argv[5]); 12 | count.resize(sumAli, 0); 13 | infile >> term; 14 | while (!infile.eof()) { 15 | count[term]++; 16 | sumNum++; 17 | infile >> term; 18 | } 19 | infile.close(); 20 | outfile<<" ["; 21 | for (int i = 0; i < count.size(); i++) { 22 | outfile << " " << double(count[i])/sumNum; 23 | } 24 | outfile << " ]" << std::endl; 25 | outfile2 << sumNum << " " << input_dim << " " << sumAli << std::endl; 26 | outfile.close(); 27 | outfile2.close(); 28 | return 0; 29 | } 30 | 31 | -------------------------------------------------------------------------------- /app/dnn_speech/petuumdnn/ark2txt.cc: -------------------------------------------------------------------------------- 1 | #include "base/kaldi-common.h" 2 | #include "util/common-utils.h" 3 | #include "hmm/transition-model.h" 4 | #include "nnet2/train-nnet.h" 5 | #include "nnet2/am-nnet.h" 6 | #include "nnet2/nnet-example-functions.h" 7 | #include "nnet2/nnet-example.h" 8 | #include "matrix/kaldi-matrix.h" 9 | #include 10 | 11 | int main(int argc, char *argv[]) { 12 | try { 13 | using namespace kaldi; 14 | std::string rspecifier = argv[1]; 15 | std::ofstream outfile("Transform.txt",std::ios::out); 16 | 17 | SequentialBaseFloatMatrixReader mat_reader(rspecifier); 18 | 19 | int32 num_done = 0; 20 | int64 num_rows_done = 0; 21 | 22 | for (; !mat_reader.Done(); mat_reader.Next()) { 23 | std::string key = mat_reader.Key(); 24 | Matrix mat(mat_reader.Value()); 25 | //outfile << mat.NumRows() << " " << mat.NumCols() << std::endl; 26 | for (int i = 0; i < mat.NumRows(); i++) { 27 | for (int j = 0; j < mat.NumCols(); j++) { 28 | outfile << mat(i,j) << " "; 29 | } 30 | } 31 | outfile << std::endl; 32 | num_done++; 33 | } 34 | 35 | KALDI_LOG << "Summed rows " << num_done << " matrices, " 36 | << num_rows_done << " rows in total."; 37 | 38 | return (num_done != 0 ? 0 : 1); 39 | } catch(const std::exception &e) { 40 | std::cerr << e.what(); 41 | return -1; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /app/dnn_speech/petuumdnn/asc2bin.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | int main(int argc, char **argv) { 5 | std::ifstream infile(argv[1],std::ios::in); 6 | std::string datafile = std::string(argv[1])+".Bin"; 7 | std::ofstream outfile(datafile.c_str(),std::ios::binary|std::ios::out); 8 | float term; 9 | infile >> term; 10 | while (!infile.eof()) { 11 | outfile.write((char *)&term, sizeof(float)); 12 | infile >> term; 13 | } 14 | infile.close(); 15 | outfile.close(); 16 | return 0; 17 | } -------------------------------------------------------------------------------- /app/dnn_speech/petuumdnn/bin2asc.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | int main(int argc, char **argv) { 6 | std::ifstream infile(argv[1],std::ios::binary|std::ios::in); 7 | std::string datafile = std::string(argv[1])+".ASC"; 8 | std::ofstream outfile(datafile.c_str(),std::ios::out); 9 | float term; int dim = atoi(argv[2]); 10 | int count = 0; 11 | infile.read((char *)&term, sizeof(float)); 12 | while (!infile.eof()) { 13 | count++; 14 | outfile << term << " "; 15 | if (count == dim) { 16 | outfile << std::endl; 17 | count = 0; 18 | } 19 | infile.read((char *)&term, sizeof(float)); 20 | } 21 | infile.close(); 22 | outfile.close(); 23 | return 0; 24 | } -------------------------------------------------------------------------------- /app/dnn_speech/petuumdnn/check.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | int main(int argc, char **argv) { 6 | double term1, term2, max = -1; 7 | std::ifstream infile1("Transform.txt",std::ios::in); 8 | std::ifstream infile2("Train.fea.1",std::ios::in); 9 | std::ofstream outfile("check.result", std::ios::out); 10 | infile1 >> term1; infile2 >> term2; 11 | while (!infile1.eof()) { 12 | if (fabs(term1-term2) > max) max = fabs(term1-term2); 13 | infile1 >> term1; infile2 >> term2; 14 | } 15 | outfile << max; 16 | infile1.close(); 17 | infile2.close(); 18 | outfile.close(); 19 | return 0; 20 | } -------------------------------------------------------------------------------- /app/dnn_speech/petuumdnn/parse_head.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | int main(int argc, char **argv) { 7 | std::ifstream infile(argv[1], std::ios::in); 8 | std::ofstream outfile(argv[2], std::ios::out); 9 | std::ofstream outfile2(argv[3], std::ios::out); 10 | std::string line, num; 11 | int t, t2, t3, t4; 12 | bool flag = false; 13 | std::getline(infile, line); 14 | while (!infile.eof()) { 15 | t = line.find(""); 16 | if ( t != std::string::npos ) { 17 | std::string head = line.substr(0, t+16); 18 | t2 = line.find(""); 19 | std::string tail = line.substr(t2); 20 | outfile << head << " " << 0 << " " << tail << std::endl; 21 | } else { 22 | t2 = line.find(""); 23 | if ( t2 != std::string::npos ) { 24 | outfile << line << std::endl; 25 | break; 26 | } 27 | outfile << line << std::endl; 28 | } 29 | if (flag == true) { 30 | t3 = line.find("["); 31 | t4 = line.find("]"); 32 | int beg = 0, len = -1; 33 | if (t3 != std::string::npos) beg = t3+1; 34 | if (t4 != std::string::npos) len = t4 - t3 - 1; 35 | if (len != -1) outfile2 << line.substr(beg, len) << std::endl; else outfile2 << line.substr(beg) << std::endl; 36 | } 37 | if ( line.find("") != std::string::npos ) flag = true; 38 | std::getline(infile, line); 39 | } 40 | infile.close(); 41 | outfile2.close(); 42 | outfile.close(); 43 | return 0; 44 | } -------------------------------------------------------------------------------- /app/dnn_speech/petuumdnn/set_num_layers.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | int main(int argc, char **argv) { 7 | std::ifstream infile(argv[1], std::ios::in); 8 | std::ifstream infile2(argv[3], std::ios::in); 9 | std::ofstream outfile(argv[2], std::ios::out); 10 | std::string line, num; 11 | int num_layers, t, t2; 12 | std::getline(infile2, line); 13 | t = line.find(":"); 14 | num = line.substr(t+1); 15 | sscanf(num.c_str(), "%d", &num_layers); 16 | std::getline(infile, line); 17 | while (!infile.eof()) { 18 | t = line.find(""); 19 | if ( t != std::string::npos ) { 20 | std::string head = line.substr(0, t+16); 21 | t2 = line.find(""); 22 | std::string tail = line.substr(t2); 23 | outfile << head << " " << num_layers*2 << " " << tail << std::endl; 24 | } else { 25 | t2 = line.find(""); 26 | if ( t2 != std::string::npos ) { 27 | outfile << line << std::endl; 28 | break; 29 | } 30 | outfile << line << std::endl; 31 | } 32 | std::getline(infile, line); 33 | } 34 | infile.close(); 35 | infile2.close(); 36 | outfile.close(); 37 | return 0; 38 | } -------------------------------------------------------------------------------- /app/dnn_speech/petuumdnn/testcatright.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | int main(int argc, char **argv) { 6 | int num = 30; 7 | std::ostringstream ss; 8 | std::ofstream outfile("testout.txt",std::ios::out); 9 | std::ofstream outfile2("testout2.txt",std::ios::out); 10 | float term; 11 | for (int i = 1; i <= num; i++) { 12 | ss << i; 13 | std::string feaname = "Train.fea."+ss.str(); 14 | //std::string labelname = "Train.label."+ss.str(); 15 | std::ifstream infile(feaname.c_str(),std::ios::binary); 16 | infile.read((char *)&term, sizeof(float)); 17 | while (!infile.eof()) { 18 | outfile << term << " "; 19 | infile.read((char *)&term, sizeof(float)); 20 | } 21 | infile.close(); 22 | } 23 | std::ifstream infile2("Train.fea",std::ios::binary); 24 | infile2.read((char *)&term, sizeof(float)); 25 | while (!infile2.eof()) { 26 | outfile2 << term << " "; 27 | infile2.read((char *)&term, sizeof(float)); 28 | } 29 | infile2.close(); 30 | outfile.close(); 31 | outfile2.close(); 32 | return 0; 33 | } -------------------------------------------------------------------------------- /app/dnn_speech/scripts/NetworkDecode.sh: -------------------------------------------------------------------------------- 1 | DNN_para=$1 2 | para_file=$2 3 | 4 | cp kaldi-trunk/src/petuumdnn/set_num_layers ./ 5 | mv head.txt oldhead.txt 6 | ./set_num_layers oldhead.txt head.txt $para_file 7 | cat head.txt $DNN_para tail.txt > final.mdl 8 | rm tail.txt 9 | rm oldhead.txt 10 | rm head.txt 11 | rm set_num_layers 12 | cp final.mdl kaldi-trunk/egs/timit/s5/exp/petuum_dnn/ 13 | cd kaldi-trunk/egs/timit/s5/ 14 | scripts/adjust_prior.sh data/train data/lang exp/tri3_ali exp/petuum_dnn 15 | cp scripts/timit_decode.sh ./ 16 | ./timit_decode.sh 17 | rm timit_decode.sh 18 | -------------------------------------------------------------------------------- /app/dnn_speech/scripts/PrepDNNFeature.sh: -------------------------------------------------------------------------------- 1 | timit_path=$1 2 | 3 | cp -r scripts kaldi-trunk/egs/timit/s5/ 4 | cp -r petuumdnn/ kaldi-trunk/src/ 5 | cd kaldi-trunk/src/petuumdnn/ 6 | make 7 | cd ../../egs/timit/s5/ 8 | mv scripts/path.sh ./ 9 | mv scripts/timit_data_prep.sh ./ 10 | #mv scripts/timit.sh ./ 11 | cp scripts/petuum_dnn.sh ./ 12 | ./timit_data_prep.sh $timit_path 13 | #./timit.sh 14 | cp Train.fea Train.label Train.para tail.txt head.txt ../../../../ 15 | rm Train.fea Train.label Train.para tail.txt head.txt 16 | rm petuum_dnn.sh 17 | rm timit_data_prep.sh 18 | #rm timit.sh 19 | -------------------------------------------------------------------------------- /app/dnn_speech/scripts/gen_data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ $# -ne 5 ]; then 3 | echo "Usage: $0 " 4 | exit 5 | fi 6 | progname=gen_data 7 | script_path=`readlink -f $0` 8 | script_dir=`dirname $script_path` 9 | project_root=`dirname $script_dir` 10 | prog_path=$project_root/bin/$progname 11 | if [ ! -d $5 ]; then 12 | mkdir $5 13 | fi 14 | $prog_path $1 $2 $3 $4 `readlink -f $5` 15 | -------------------------------------------------------------------------------- /app/dnn_speech/scripts/kill_dnn.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ $# -ne 1 ]; then 4 | echo "Usage: $0 " 5 | echo "" 6 | echo "Kills hung DNN clients" 7 | exit 8 | fi 9 | 10 | host_file=`readlink -f $1` 11 | 12 | progname=DNN 13 | ssh_options="-oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -oLogLevel=quiet" 14 | 15 | # Parse hostfile 16 | unique_host_list=`cat $host_file | awk '{ print $2 }' | uniq` 17 | 18 | # Kill instances 19 | echo "Killing previous instances of '$progname' on servers, please wait..." 20 | for ip in $unique_host_list; do 21 | ssh $ssh_options $ip \ 22 | killall -q $progname 23 | done 24 | echo "All done!" -------------------------------------------------------------------------------- /app/dnn_speech/scripts/kill_dnn_sn.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | 4 | progname=DNN_sn 5 | 6 | killall -q $progname 7 | -------------------------------------------------------------------------------- /app/dnn_speech/scripts/path.sh: -------------------------------------------------------------------------------- 1 | # assume Kaldi is in the same directory of apps/dnn_speech now 2 | export KALDI_ROOT=`pwd`/../../.. 3 | export PATH=$PWD/scripts:$KALDI_ROOT/src/petuumdnn:$PWD/utils/:$KALDI_ROOT/src/bin:$KALDI_ROOT/tools/openfst/bin:$KALDI_ROOT/tools/irstlm/bin/:$KALDI_ROOT/src/fstbin/:$KALDI_ROOT/src/gmmbin/:$KALDI_ROOT/src/featbin/:$KALDI_ROOT/src/lm/:$KALDI_ROOT/src/sgmmbin/:$KALDI_ROOT/src/sgmm2bin/:$KALDI_ROOT/src/fgmmbin/:$KALDI_ROOT/src/latbin/:$KALDI_ROOT/src/nnetbin:$KALDI_ROOT/src/nnet2bin/:$KALDI_ROOT/src/kwsbin:$PWD:$PATH 4 | export LC_ALL=C 5 | export IRSTLM=$KALDI_ROOT/tools/irstlm 6 | -------------------------------------------------------------------------------- /app/dnn_speech/scripts/run_dnn_sn.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ $# -lt 5 -o $# -gt 6 ]; then 4 | echo "Usage: $0 \"additional options\"" 5 | exit 6 | fi 7 | 8 | 9 | num_worker_threads=$1; 10 | staleness=$2; 11 | parafile=$(readlink -f $3) 12 | data_ptt_file=$(readlink -f $4) 13 | model_para_file=$(readlink -f $5) 14 | add_options=$6 15 | 16 | progname=DNN_sn 17 | ssh_options="-oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -oLogLevel=quiet" 18 | 19 | # Find other Petuum paths by using the script's path 20 | script_path=`readlink -f $0` 21 | script_dir=`dirname $script_path` 22 | project_root=`dirname $script_dir` 23 | prog_path=$project_root/bin/$progname 24 | 25 | # Kill previous instances of this program 26 | echo "Killing previous instances of '$progname', please wait..." 27 | killall -q $progname 28 | echo "All done!" 29 | 30 | # Create temp directory for OOC 31 | ooc_dir=dnn_sn.localooc 32 | echo "Creating temp directory for PS out-of-core: $ooc_dir" 33 | rm -rf $ooc_dir 34 | mkdir $ooc_dir 35 | 36 | echo $parafile 37 | # Run single-machine program 38 | $prog_path \ 39 | --num_clients 1 \ 40 | --client_id 0 \ 41 | --staleness $staleness \ 42 | --num_worker_threads $num_worker_threads \ 43 | --data_ptt_file $data_ptt_file \ 44 | --model_para_file $model_para_file \ 45 | $add_options \ 46 | --parafile $parafile & 47 | -------------------------------------------------------------------------------- /app/kmeans/Makefile: -------------------------------------------------------------------------------- 1 | APP_DIR := $(shell readlink $(dir $(lastword $(MAKEFILE_LIST))) -f) 2 | PETUUM_ROOT = $(APP_DIR)/../../ 3 | 4 | include $(PETUUM_ROOT)/defns.mk 5 | 6 | APP_SRC = $(wildcard $(APP_DIR)/src/*.cpp) 7 | APP_HDR = $(wildcard $(APP_DIR)/src/*.h) 8 | 9 | 10 | APP_BIN = $(APP_DIR)/bin 11 | APP_OBJ = $(APP_SRC:.cpp=.o) 12 | NDEBUG = -DNDEBUG 13 | 14 | all: $(APP_BIN)/kmeans_main 15 | 16 | $(APP_BIN): 17 | mkdir -p $(APP_BIN) 18 | 19 | $(APP_BIN)/kmeans_main: $(APP_OBJ) $(PETUUM_PS_LIB) $(PETUUM_ML_LIB) $(APP_BIN) 20 | $(PETUUM_CXX) $(PETUUM_CXXFLAGS) $(PETUUM_INCFLAGS) \ 21 | $(APP_OBJ) $(PETUUM_PS_LIB) $(PETUUM_ML_LIB) $(PETUUM_LDFLAGS) -o $@ 22 | 23 | $(APP_OBJ): %.o: %.cpp $(APP_HDR) 24 | $(PETUUM_CXX) $(NDEBUG) $(PETUUM_CXXFLAGS) -Wno-unused-result \ 25 | $(PETUUM_INCFLAGS) -c $< -o $@ 26 | 27 | 28 | clean: 29 | rm -rf $(APP_OBJ) 30 | rm -rf $(APP_BIN) 31 | 32 | .PHONY: clean 33 | -------------------------------------------------------------------------------- /app/kmeans/README.md: -------------------------------------------------------------------------------- 1 | #Quick start 2 | 3 | Compile and generate datasets: 4 | ``` 5 | make -j2 6 | mkdir -p datasets 7 | cd datasets 8 | python ../scripts/data_generate.py 100 100 10000 libsvm 9 | ``` 10 | then put them to HDFS. 11 | ``` 12 | hadoop fs -mkdir -p /dataset/kmeans 13 | hadoop fs -put * /dataset/kmeans/ 14 | ``` 15 | run kmeans without yarn: 16 | ``` 17 | cp scripts/run_kmeans.py.template scripts/run_kmeans.py 18 | python scripts/run_kmeans.py 19 | ``` 20 | To run kmeans on yarn, build `src/yarn` under repo root first (if not done 21 | already): 22 | ``` 23 | cd ../../src/yarn 24 | gradle build 25 | ``` 26 | Then come back to `app/kmeans` and do 27 | ``` 28 | cp scripts/launch_yarn_job.py.template scripts/launch_yarn_job.py 29 | cp scripts/run_kmeans_on_yarn.py.template scripts/run_kmeans_on_yarn.py 30 | # Change the HDFS path in scripts/run_kmeans_on_yarn.py if you put data under 31 | # a path different from the default (/dataset/kmeans) 32 | python scripts/launch_yarn_job.py 33 | ``` 34 | 35 | The results should be in `output/` in local FS or `hdfs:///tmp/kmeans/` in 36 | HDFS. 37 | -------------------------------------------------------------------------------- /app/kmeans/script/kill.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os, sys 4 | 5 | if len(sys.argv) != 2: 6 | print "usage: %s " % sys.argv[0] 7 | sys.exit(1) 8 | 9 | host_file = sys.argv[1] 10 | prog_name = "kmeans_main" 11 | 12 | # Get host IPs 13 | with open(host_file, "r") as f: 14 | hostlines = f.read().splitlines() 15 | host_ips = [line.split()[1] for line in hostlines] 16 | 17 | ssh_cmd = ( 18 | "ssh " 19 | "-o StrictHostKeyChecking=no " 20 | "-o UserKnownHostsFile=/dev/null " 21 | ) 22 | 23 | for ip in host_ips: 24 | cmd = ssh_cmd + ip + " killall -q " + prog_name 25 | os.system(cmd) 26 | print "Done killing" 27 | -------------------------------------------------------------------------------- /app/kmeans/script/launch_on_yarn.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | from os.path import dirname 5 | from os.path import join 6 | import time 7 | 8 | app_dir = dirname(dirname(os.path.realpath(__file__))) 9 | proj_dir = dirname(dirname(app_dir)) 10 | 11 | params = { 12 | "jar": join(proj_dir, "src/yarn/build/libs/yarn-0.5.jar") 13 | , "launch_script_path": join(app_dir, "script", "run_local.py") 14 | , "container_memory": 500 15 | , "container_vcores": 1 16 | , "master_memory": 350 17 | , "priority": 10 18 | , "num_nodes": 1 19 | } 20 | 21 | cmd = "hadoop jar " 22 | cmd += join(proj_dir, "src/yarn/build/libs/yarn-0.5.jar") + " " 23 | cmd += "cn.edu.buaa.act.petuumOnYarn.Client" 24 | cmd += "".join([" --%s %s" % (k,v) for k,v in params.items()]) 25 | print cmd 26 | os.system(cmd) 27 | -------------------------------------------------------------------------------- /app/kmeans/src/cluster_centers.h: -------------------------------------------------------------------------------- 1 | /* 2 | * clustercenters.h 3 | * 4 | * Created on: Oct 16, 2014 5 | * Author: manu 6 | */ 7 | 8 | #ifndef CLUSTERCENTERS_H_ 9 | #define CLUSTERCENTERS_H_ 10 | 11 | #include 12 | #include 13 | #include "sparse_vector.h" 14 | 15 | using std::vector; 16 | using std::string; 17 | 18 | class cluster_centers { 19 | public: 20 | cluster_centers(); 21 | cluster_centers(int dimensionality, int number_of_clusters); 22 | virtual ~cluster_centers(); 23 | void insertAtPosition(const sparse_vector& x, int position); 24 | void random_initiliaze(); 25 | const sparse_vector& getCenterAt(int position) const; 26 | float SqDistanceToClosestCenter(const sparse_vector& x, int& center_id); 27 | int NumOfCenters() const { 28 | return cluster_centers_.size(); 29 | } 30 | ; 31 | sparse_vector* getPointerToCenterAt(int position); 32 | void clear(); 33 | void setValueToFeature(int center, int featureId, float featureValue); 34 | void saveClusterCentersToDisk(string location); 35 | void LoadClusterCentersFromDisk(string location); 36 | 37 | protected: 38 | float SqDistanceToCenterI(int center_id, const sparse_vector& x); 39 | 40 | // The set of cluster centers. 41 | vector cluster_centers_; 42 | int number_of_clusters_; 43 | int dimensionality_; 44 | 45 | 46 | private: 47 | 48 | }; 49 | 50 | #endif /* CLUSTERCENTERS_H_ */ 51 | -------------------------------------------------------------------------------- /app/kmeans/src/context.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | namespace util { 9 | 10 | // An extension of google flags. It is a singleton that stores 1) google flags 11 | // and 2) other lightweight global flags. Underlying data structure is map of 12 | // string and string, similar to google::CommandLineFlagInfo. 13 | class Context { 14 | public: 15 | static Context& get_instance(); 16 | 17 | int get_int32(std::string key); 18 | double get_double(std::string key); 19 | bool get_bool(std::string key); 20 | std::string get_string(std::string key); 21 | 22 | void set(std::string key, int value); 23 | void set(std::string key, double value); 24 | void set(std::string key, bool value); 25 | void set(std::string key, std::string value); 26 | 27 | private: 28 | // Private constructor. Store all the gflags values. 29 | Context(); 30 | // Underlying data structure 31 | std::unordered_map ctx_; 32 | }; 33 | 34 | } // namespace util 35 | -------------------------------------------------------------------------------- /app/kmeans/src/dataset.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * dataset.cpp 3 | * 4 | * Created on: Oct 17, 2014 5 | * Author: manu 6 | */ 7 | 8 | #include "dataset.h" 9 | #include "assert.h" 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include "io/general_fstream.hpp" 17 | #include 18 | 19 | using std::string; 20 | using std::cout; 21 | using std::endl; 22 | 23 | dataset::dataset() { 24 | } 25 | 26 | dataset::~dataset() { 27 | dataset_.clear(); 28 | } 29 | 30 | dataset::dataset(const string& file_name, int buffer_mb, int start_index, int end_index) { 31 | petuum::io::ifstream file_stream(file_name); 32 | CHECK(file_stream) << "Failed to open " << file_name; 33 | 34 | string line_string; 35 | int line_count=0; 36 | while (getline(file_stream, line_string)) { 37 | 38 | if(line_count>=start_index && line_count < end_index){ 39 | AddDataPoint(line_string); 40 | } 41 | line_count++; 42 | } 43 | } 44 | 45 | int dataset::Size() const{ 46 | return dataset_.size(); 47 | } 48 | 49 | const sparse_vector dataset::getDataPointAt(long x) const { 50 | assert(x 15 | 16 | using std::vector; 17 | using std::string; 18 | 19 | class dataset { 20 | public: 21 | dataset(); 22 | virtual ~dataset(); 23 | int Size() const; 24 | dataset(const string& file_name, int buffer_mb, int start_index, int end_index); 25 | const sparse_vector getDataPointAt(long x) const; 26 | // Adds the vector represented by this svm-light format string 27 | // to the data set. 28 | void AddDataPoint(const string& vector_string); 29 | void AddDataPoint(const char* vector_string); 30 | // Adds a copy of the given vector, using label y. 31 | 32 | private: 33 | vector dataset_; 34 | }; 35 | 36 | #endif /* DATASET_H_ */ 37 | -------------------------------------------------------------------------------- /app/kmeans/src/dense_vector.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * dense_vector.cpp 3 | * 4 | * Created on: Oct 18, 2014 5 | * Author: manu 6 | */ 7 | 8 | #include "dense_vector.h" 9 | 10 | dense_vector::dense_vector() { 11 | // TODO Auto-generated constructor stub 12 | 13 | } 14 | 15 | dense_vector::~dense_vector() { 16 | // TODO Auto-generated destructor stub 17 | } 18 | 19 | -------------------------------------------------------------------------------- /app/kmeans/src/dense_vector.h: -------------------------------------------------------------------------------- 1 | /* 2 | * dense_vectorh.h 3 | * 4 | * Created on: Oct 18, 2014 5 | * Author: manu 6 | */ 7 | 8 | #ifndef DENSE_VECTOR_H_ 9 | #define DENSE_VECTOR_H_ 10 | 11 | class dense_vector { 12 | public: 13 | dense_vector(); 14 | virtual ~dense_vector(); 15 | 16 | 17 | private: 18 | float * center; 19 | 20 | }; 21 | 22 | #endif /* DENSE_VECTOR_H_ */ 23 | -------------------------------------------------------------------------------- /app/kmeans/src/kmeans_methods.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * KmeansMethods.cpp 3 | * 4 | * Created on: Oct 18, 2014 5 | * Author: manu 6 | */ 7 | 8 | #include "kmeans_methods.h" 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include "dataset.h" 15 | #include 16 | 17 | 18 | int randInt(int num) { 19 | int i = static_cast(rand()) % num; 20 | if (i < 0) { 21 | i = i + num; 22 | } 23 | return i; 24 | } 25 | 26 | void RandomInitializeCenters(cluster_centers* centers, const dataset& dataset1) { 27 | std::random_device rd; 28 | std::uniform_int_distribution dist(0, dataset1.Size()-1); 29 | 30 | assert(centers->NumOfCenters() > 0 && centers->NumOfCenters() < dataset1.Size()); 31 | for (int i = 0; i < centers->NumOfCenters(); i++) { 32 | int p = dist(rd); 33 | centers->insertAtPosition(dataset1.getDataPointAt(p), i); 34 | } 35 | } 36 | 37 | float ComputeObjective(cluster_centers* centers, const dataset& ds){ 38 | int center_id; 39 | float total_sq_distance = 0.0; 40 | for (int i = 0; i < ds.Size(); ++i) { 41 | total_sq_distance += centers->SqDistanceToClosestCenter(ds.getDataPointAt(i),center_id); 42 | } 43 | return total_sq_distance; 44 | } 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /app/kmeans/src/kmeans_methods.h: -------------------------------------------------------------------------------- 1 | /* 2 | * KMEANSMethods.h 3 | * 4 | * Created on: Oct 18, 2014 5 | * Author: manu 6 | */ 7 | 8 | #ifndef KMEANSMETHODS_H_ 9 | #define KMEANSMETHODS_H_ 10 | 11 | #include "cluster_centers.h" 12 | #include "sparse_vector.h" 13 | #include "dataset.h" 14 | 15 | int randInt(int num); 16 | void RandomInitializeCenters(cluster_centers* centers, const dataset& dataset1); 17 | float ComputeObjective(cluster_centers* centers, const dataset& ds); 18 | 19 | #endif /* KMEANSMETHODS_H_ */ 20 | -------------------------------------------------------------------------------- /app/kmeans/src/sparse_vector.h: -------------------------------------------------------------------------------- 1 | /* 2 | * sparse_vector.h 3 | * 4 | * Created on: Oct 16, 2014 5 | * Author: manu 6 | */ 7 | 8 | #ifndef SPARSE_VECTOR_H_ 9 | #define SPARSE_VECTOR_H_ 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | 16 | using std::vector; 17 | using std::string; 18 | 19 | 20 | struct FeatureValuePair { 21 | int id_; 22 | float value_; 23 | }; 24 | 25 | class sparse_vector { 26 | public: 27 | sparse_vector(); 28 | sparse_vector(const char* input); 29 | sparse_vector(int dimensionality, bool isSparse); 30 | sparse_vector(const sparse_vector& s, bool isSparse); 31 | sparse_vector(const sparse_vector& s); 32 | virtual ~sparse_vector(); 33 | void push_back(int id, float value); 34 | int FeatureAt (int i) const; 35 | float ValueAt (int i) const; 36 | int size() const; 37 | string AsString() const; 38 | float getSquareNorm() const{return square_norm;}; 39 | void reComputeSquaredNorm(); 40 | float getInnerProduct(const sparse_vector & x); 41 | void scaleBy(float eta); 42 | void Add(const sparse_vector& x, float eta ); 43 | void set(int featureId, float featureValue); 44 | 45 | private: 46 | vector vectors_; 47 | bool isSparse_; 48 | float square_norm; 49 | }; 50 | 51 | #endif /* SPARSE_VECTOR_H_ */ 52 | -------------------------------------------------------------------------------- /app/mlr/Makefile: -------------------------------------------------------------------------------- 1 | MLR_DIR := $(shell readlink $(dir $(lastword $(MAKEFILE_LIST))) -f) 2 | PETUUM_ROOT = $(MLR_DIR)/../../ 3 | 4 | include $(PETUUM_ROOT)/defns.mk 5 | 6 | MLR_SRC = $(wildcard $(MLR_DIR)/src/*.cpp) 7 | MLR_HDR = $(wildcard $(MLR_DIR)/src/*.hpp) 8 | MLR_BIN = $(MLR_DIR)/bin 9 | MLR_OBJ = $(MLR_SRC:.cpp=.o) 10 | NDEBUG = -DNDEBUG 11 | 12 | all: $(MLR_BIN)/mlr_main $(MLR_BIN)/gen_data_sparse 13 | 14 | $(MLR_BIN): 15 | mkdir -p $(MLR_BIN) 16 | 17 | $(MLR_BIN)/mlr_main: $(MLR_OBJ) $(PETUUM_PS_LIB) $(PETUUM_ML_LIB) $(MLR_BIN) 18 | $(PETUUM_CXX) $(PETUUM_CXXFLAGS) $(PETUUM_INCFLAGS) \ 19 | $(MLR_OBJ) $(PETUUM_PS_LIB) $(PETUUM_ML_LIB) $(PETUUM_LDFLAGS) -o $@ 20 | 21 | $(MLR_OBJ): %.o: %.cpp $(MLR_HDR) 22 | $(PETUUM_CXX) $(NDEBUG) $(PETUUM_CXXFLAGS) -Wno-unused-result \ 23 | $(PETUUM_INCFLAGS) -c $< -o $@ 24 | 25 | $(MLR_BIN)/gen_data_sparse: $(MLR_DIR)/src/tools/gen_data_sparse.cpp $(MLR_BIN) 26 | $(PETUUM_CXX) $(PETUUM_CXXFLAGS) $(PETUUM_INCFLAGS) \ 27 | $< $(PETUUM_PS_LIB) $(PETUUM_LDFLAGS) -o $@ 28 | 29 | clean: 30 | rm -rf $(MLR_OBJ) 31 | rm -rf $(MLR_BIN) 32 | 33 | .PHONY: clean 34 | -------------------------------------------------------------------------------- /app/mlr/datasets/covtype.scale.test.small.meta: -------------------------------------------------------------------------------- 1 | num_data: 50 2 | num_test: 50 3 | feature_dim: 54 4 | num_labels: 7 5 | format: libsvm 6 | feature_one_based: 1 7 | label_one_based: 1 8 | snappy_compressed: 0 9 | -------------------------------------------------------------------------------- /app/mlr/datasets/covtype.scale.train.small.meta: -------------------------------------------------------------------------------- 1 | num_train_total: 500 2 | num_train_this_partition: 500 3 | feature_dim: 54 4 | num_labels: 7 5 | format: libsvm 6 | feature_one_based: 1 7 | label_one_based: 1 8 | snappy_compressed: 0 9 | -------------------------------------------------------------------------------- /app/mlr/script/kill.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os, sys 4 | 5 | if len(sys.argv) != 2: 6 | print "usage: %s " % sys.argv[0] 7 | sys.exit(1) 8 | 9 | host_file = sys.argv[1] 10 | prog_name = "mlr_main" 11 | 12 | # Get host IPs 13 | with open(host_file, "r") as f: 14 | hostlines = f.read().splitlines() 15 | host_ips = [line.split()[1] for line in hostlines] 16 | 17 | ssh_cmd = ( 18 | "ssh " 19 | "-o StrictHostKeyChecking=no " 20 | "-o UserKnownHostsFile=/dev/null " 21 | ) 22 | 23 | for ip in host_ips: 24 | cmd = ssh_cmd + ip + " killall -q " + prog_name 25 | os.system(cmd) 26 | print "Done killing" 27 | -------------------------------------------------------------------------------- /app/mlr/script/launch_on_yarn.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | from os.path import dirname 5 | from os.path import join 6 | import time 7 | 8 | app_dir = dirname(dirname(os.path.realpath(__file__))) 9 | proj_dir = dirname(dirname(app_dir)) 10 | 11 | params = { 12 | "jar": join(proj_dir, "src/yarn/build/libs/yarn-0.5.jar") 13 | , "launch_script_path": join(app_dir, "script", "run_local.py") 14 | , "container_memory": 500 15 | , "container_vcores": 1 16 | , "master_memory": 350 17 | , "priority": 10 18 | , "num_nodes": 1 19 | } 20 | 21 | cmd = "hadoop jar " 22 | cmd += join(proj_dir, "src/yarn/build/libs/yarn-0.5.jar") + " " 23 | cmd += "cn.edu.buaa.act.petuumOnYarn.Client" 24 | cmd += "".join([" --%s %s" % (k,v) for k,v in params.items()]) 25 | print cmd 26 | os.system(cmd) 27 | -------------------------------------------------------------------------------- /app/mlr/script/run_gen_data.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | from os.path import dirname 5 | from os.path import join 6 | 7 | app_dir = dirname(dirname(os.path.realpath(__file__))) 8 | prog = join(app_dir, "bin", "gen_data_sparse") 9 | 10 | # no trailing / 11 | prefix_path = join(app_dir, "datasets") 12 | 13 | params = { 14 | "num_train": 100 15 | , "feature_dim": 10 16 | , "num_partitions": 1 17 | , "nnz_per_col": 10 18 | , "one_based": True 19 | , "beta_sparsity": 0.5 20 | , "correlation_strength": 0.5 21 | , "noise_ratio": 0.1 22 | , "snappy_compressed": "false" 23 | , "num_labels": 2 24 | } 25 | params["output_file"] = join(prefix_path, "lr%d_dim%d_s%d_nnz%d") \ 26 | % (params["num_labels"], params["feature_dim"], \ 27 | params["num_train"], params["nnz_per_col"]) 28 | 29 | 30 | env_params = ( 31 | "GLOG_logtostderr=true " 32 | "GLOG_v=-1 " 33 | "GLOG_minloglevel=0 " 34 | ) 35 | 36 | cmd = env_params + prog 37 | cmd += "".join([" --%s=%s" % (k,v) for k,v in params.items()]) 38 | print cmd 39 | os.system(cmd) 40 | -------------------------------------------------------------------------------- /app/mlr/src/common.hpp: -------------------------------------------------------------------------------- 1 | // Author: Dai Wei (wdai@cs.cmu.edu) 2 | // Date: 2014.07.14 3 | 4 | #pragma once 5 | 6 | #include 7 | #include 8 | 9 | // Globally accessible gflags. 10 | DECLARE_int32(num_clients); 11 | DECLARE_int32(num_app_threads); 12 | DECLARE_int32(client_id); 13 | DECLARE_int32(num_comm_channels_per_client); 14 | DECLARE_int32(num_train_data); 15 | DECLARE_string(train_file); 16 | DECLARE_bool(global_data); 17 | DECLARE_string(test_file); 18 | DECLARE_int32(num_train_eval); 19 | DECLARE_int32(num_test_eval); 20 | DECLARE_bool(perform_test); 21 | DECLARE_bool(use_weight_file); 22 | DECLARE_string(weight_file); 23 | 24 | DECLARE_int32(num_epochs); 25 | DECLARE_int32(num_batches_per_epoch); 26 | DECLARE_double(init_lr); 27 | DECLARE_double(lr_decay_rate); 28 | DECLARE_int32(num_batches_per_eval); 29 | DECLARE_bool(sparse_weight); 30 | DECLARE_double(lambda); 31 | 32 | DECLARE_string(output_file_prefix); 33 | DECLARE_int32(w_table_id); 34 | DECLARE_int32(loss_table_id); 35 | DECLARE_int32(staleness); 36 | DECLARE_int32(num_secs_per_checkpoint); 37 | DECLARE_int32(w_table_num_cols); 38 | 39 | 40 | namespace mlr { 41 | 42 | const int32_t kColIdxLossTableEpoch = 0; 43 | const int32_t kColIdxLossTableBatch = 1; 44 | const int32_t kColIdxLossTableZeroOneLoss = 2; // training set 45 | const int32_t kColIdxLossTableEntropyLoss = 3; // training set 46 | const int32_t kColIdxLossTableNumEvalTrain = 4; // # train point eval 47 | const int32_t kColIdxLossTableTestZeroOneLoss = 5; 48 | const int32_t kColIdxLossTableNumEvalTest = 6; // # test point eval. 49 | const int32_t kColIdxLossTableTime = 7; 50 | 51 | const int32_t kNumColumnsLossTable = 8; 52 | 53 | } // namespace mlr 54 | -------------------------------------------------------------------------------- /app/rand_forest/Makefile: -------------------------------------------------------------------------------- 1 | RF_DIR := $(shell readlink $(dir $(lastword $(MAKEFILE_LIST))) -f) 2 | PETUUM_ROOT = $(RF_DIR)/../../ 3 | RF_SRC_DIR = $(RF_DIR)/src 4 | RF_TEST_DIR = $(RF_DIR)/test 5 | RF_TEST_INCFLAGS = $(PETUUM_INCFLAGS) -I$(RF_SRC_DIR) 6 | RF_TEST_LDFLAGS = $(PETUUM_LDFLAGS) -lgtest_main 7 | 8 | include $(PETUUM_ROOT)/defns.mk 9 | 10 | RF_SRC = $(wildcard $(RF_SRC_DIR)/*.cpp) 11 | RF_HDR = $(wildcard $(RF_SRC_DIR)/*.hpp) 12 | 13 | RF_BIN = $(RF_DIR)/bin 14 | RF_OBJ = $(RF_SRC:.cpp=.o) 15 | 16 | all: $(RF_BIN)/rand_forest_main 17 | 18 | test: split_finder_test_run 19 | 20 | $(RF_BIN): 21 | mkdir -p $(RF_BIN) 22 | 23 | $(RF_BIN)/rand_forest_main: $(RF_OBJ) $(PETUUM_PS_LIB) \ 24 | $(PETUUM_ML_LIB) $(RF_BIN) 25 | $(PETUUM_CXX) $(PETUUM_CXXFLAGS) $(PETUUM_INCFLAGS) \ 26 | $(RF_OBJ) $(PETUUM_IO_LIB) $(PETUUM_ML_LIB) $(PETUUM_PS_LIB) $(PETUUM_LDFLAGS) -o $@ 27 | 28 | $(RF_OBJ): %.o: %.cpp $(RF_HDR) 29 | $(PETUUM_CXX) $(PETUUM_CXXFLAGS) -Wno-unused-result \ 30 | $(PETUUM_INCFLAGS) -c $< -o $@ 31 | 32 | # ============= Tests ============= 33 | 34 | $(RF_BIN)/split_finder_test: $(RF_TEST_DIR)/split_finder_test.cpp \ 35 | $(RF_SRC_DIR)/split_finder.hpp $(RF_SRC_DIR)/split_finder.o \ 36 | $(RF_SRC_DIR)/utils.hpp $(RF_SRC_DIR)/utils.cpp 37 | $(PETUUM_CXX) $(PETUUM_CXXFLAGS) $(RF_TEST_INCFLAGS) $^ \ 38 | $(RF_TEST_LDFLAGS) -o $@ 39 | 40 | split_finder_test_run: $(RF_BIN)/split_finder_test 41 | $< 42 | 43 | clean: 44 | rm -rf $(RF_OBJ) 45 | rm -rf $(RF_BIN) 46 | 47 | .PHONY: clean 48 | -------------------------------------------------------------------------------- /app/rand_forest/ReadMe.md: -------------------------------------------------------------------------------- 1 | #Quick start 2 | 3 | the datasets are regenerated. 4 | put datasets to HDFS. 5 | ``` 6 | hadoop fs -mkdir -p /dataset/rand_forest 7 | hadoop fs -put datasets/* /dataset/rand_forest/ 8 | ``` 9 | run rf without yarn: 10 | ``` 11 | cp scripts/run_rf.py.template scripts/run_rf.py 12 | python scripts/run_rf.py 13 | ``` 14 | run rf on yarn: 15 | ``` 16 | cp scripts/launch_yarn_job.py.template scripts/launch_yarn_job.py 17 | cp scripts/run_rf_on_yarn.py.tmplate scripts/run_rf_on_yarn.py 18 | python scripts/launch_yarn_job.py 19 | ``` 20 | run rf_load without yarn: 21 | ``` 22 | cp scripts/run_rf_load.py.template scripts/run_rf_load.py 23 | python scripts/run_rf_load.py 24 | # the result will be stored in `output` in local FS 25 | ``` 26 | 27 | run rf_load on yarn: 28 | ``` 29 | cp scripts/launch_yarn_job.py.template scripts/launch_yarn_job.py 30 | #change the launch_script_path to run_rf_load_on_yarn.py in scripts/launch_yarn_job.py 31 | cp scripts/run_rf_load_on_yarn.py.tmplate scripts/run_rf_load_on_yarn.py 32 | python scripts/launch_yarn_job.py 33 | ``` 34 | Note: 35 | 36 | 1. Before launching the yarn job, the yarn module in PETUUM_ROOT/src/yarn must be built. 37 | 2. Result will be stored in `output` in local FS or `/tmp/rf_output/` in HDFS 38 | -------------------------------------------------------------------------------- /app/rand_forest/dataset/iris.test: -------------------------------------------------------------------------------- 1 | 1 0:6.1 1:2.8 2:4.7 3:1.2 2 | 1 0:5.0 1:2.3 2:3.3 3:1.0 3 | 0 0:5.5 1:4.2 2:1.4 3:0.2 4 | 2 0:5.8 1:2.7 2:5.1 3:1.9 5 | 1 0:6.4 1:2.9 2:4.3 3:1.3 6 | 0 0:4.8 1:3.4 2:1.9 3:0.2 7 | 2 0:7.7 1:2.6 2:6.9 3:2.3 8 | 1 0:5.5 1:2.4 2:3.8 3:1.1 9 | 0 0:5.4 1:3.9 2:1.3 3:0.4 10 | 2 0:6.7 1:3.0 2:5.2 3:2.3 11 | 0 0:4.7 1:3.2 2:1.3 3:0.2 12 | 2 0:7.7 1:3.0 2:6.1 3:2.3 13 | 1 0:6.0 1:2.9 2:4.5 3:1.5 14 | 1 0:6.6 1:2.9 2:4.6 3:1.3 15 | 1 0:5.7 1:2.9 2:4.2 3:1.3 16 | 2 0:6.7 1:3.3 2:5.7 3:2.1 17 | 1 0:6.6 1:3.0 2:4.4 3:1.4 18 | 2 0:6.0 1:2.2 2:5.0 3:1.5 19 | 1 0:5.6 1:2.7 2:4.2 3:1.3 20 | 2 0:6.5 1:3.0 2:5.2 3:2.0 21 | 1 0:6.7 1:3.0 2:5.0 3:1.7 22 | 1 0:5.0 1:2.0 2:3.5 3:1.0 23 | 1 0:5.8 1:2.7 2:4.1 3:1.0 24 | 0 0:4.8 1:3.0 2:1.4 3:0.3 25 | 1 0:5.7 1:2.6 2:3.5 3:1.0 26 | 1 0:5.9 1:3.2 2:4.8 3:1.8 27 | 2 0:6.3 1:3.3 2:6.0 3:2.5 28 | 1 0:5.7 1:3.0 2:4.2 3:1.2 29 | 0 0:4.9 1:3.1 2:1.5 3:0.1 30 | 0 0:5.2 1:3.4 2:1.4 3:0.2 31 | -------------------------------------------------------------------------------- /app/rand_forest/dataset/iris.test.meta: -------------------------------------------------------------------------------- 1 | num_test: 30 2 | feature_dim: 4 3 | num_labels: 3 4 | format: libsvm 5 | feature_one_based: 0 6 | label_one_based: 0 7 | -------------------------------------------------------------------------------- /app/rand_forest/dataset/iris.train.meta: -------------------------------------------------------------------------------- 1 | num_data: 120 2 | num_train_this_partition: 120 3 | feature_dim: 4 4 | num_labels: 3 5 | format: libsvm 6 | feature_one_based: 0 7 | label_one_based: 0 8 | -------------------------------------------------------------------------------- /app/rand_forest/script/kill.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os, sys 4 | 5 | if len(sys.argv) != 2: 6 | print "usage: %s " % sys.argv[0] 7 | sys.exit(1) 8 | 9 | host_file = sys.argv[1] 10 | prog_name = "rand_forest_main" 11 | 12 | # Get host IPs 13 | with open(host_file, "r") as f: 14 | hostlines = f.read().splitlines() 15 | host_ips = [line.split()[1] for line in hostlines] 16 | 17 | ssh_cmd = ( 18 | "ssh " 19 | "-o StrictHostKeyChecking=no " 20 | "-o UserKnownHostsFile=/dev/null " 21 | ) 22 | 23 | for ip in host_ips: 24 | cmd = ssh_cmd + ip + " killall -q " + prog_name 25 | os.system(cmd) 26 | print "Done killing" 27 | -------------------------------------------------------------------------------- /app/rand_forest/script/launch_on_yarn.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | from os.path import dirname 5 | from os.path import join 6 | import time 7 | 8 | app_dir = dirname(dirname(os.path.realpath(__file__))) 9 | proj_dir = dirname(dirname(app_dir)) 10 | 11 | params = { 12 | "jar": join(proj_dir, "src/yarn/build/libs/yarn-0.5.jar") 13 | , "launch_script_path": join(app_dir, "script", "run_local.py") 14 | , "container_memory": 500 15 | , "container_vcores": 1 16 | , "master_memory": 350 17 | , "priority": 10 18 | , "num_nodes": 1 19 | } 20 | 21 | cmd = "hadoop jar " 22 | cmd += join(proj_dir, "src/yarn/build/libs/yarn-0.5.jar") + " " 23 | cmd += "cn.edu.buaa.act.petuumOnYarn.Client" 24 | cmd += "".join([" --%s %s" % (k,v) for k,v in params.items()]) 25 | print cmd 26 | os.system(cmd) 27 | -------------------------------------------------------------------------------- /app/rand_forest/src/common.hpp: -------------------------------------------------------------------------------- 1 | // Author: Dai Wei (wdai@cs.cmu.edu) 2 | // Date: 2014.11.06 3 | 4 | #pragma once 5 | 6 | #include 7 | 8 | // Petuum Parameters 9 | DECLARE_int32(num_clients); 10 | DECLARE_int32(num_app_threads); 11 | DECLARE_int32(client_id); 12 | DECLARE_int32(num_comm_channels_per_client); 13 | 14 | // Data Parameters 15 | DECLARE_int32(num_train_data); 16 | DECLARE_int32(num_train_data_eval); 17 | DECLARE_string(train_file); 18 | DECLARE_bool(global_data); 19 | DECLARE_string(test_file); 20 | DECLARE_bool(perform_test); 21 | 22 | // Rand Forest Parameters 23 | DECLARE_int32(test_vote_table_id); 24 | DECLARE_int32(num_trees); 25 | DECLARE_int32(max_depth); 26 | DECLARE_int32(num_data_subsample); 27 | DECLARE_int32(num_features_subsample); 28 | 29 | // Save and Load 30 | DECLARE_bool(save_pred); 31 | DECLARE_string(pred_file); 32 | DECLARE_bool(save_trees); 33 | DECLARE_string(output_file); 34 | DECLARE_bool(load_trees); 35 | DECLARE_string(input_file); 36 | -------------------------------------------------------------------------------- /app/rand_forest/src/rand_forest.hpp: -------------------------------------------------------------------------------- 1 | // Author: Jiesi Zhao (jiesizhao0423@gmail.com), Wei Dai (wdai@cs.cmu.edu) 2 | // Date: 2014.11.8 3 | 4 | #pragma once 5 | 6 | #include "common.hpp" 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "decision_tree.hpp" 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | namespace tree { 18 | 19 | struct RandForestConfig { 20 | int32_t client_id; 21 | int32_t thread_id; 22 | int32_t num_threads; 23 | int32_t num_trees; 24 | bool save_trees; 25 | DecisionTreeConfig tree_config; 26 | }; 27 | 28 | class RandForest { 29 | public: 30 | RandForest(const RandForestConfig& config); 31 | 32 | // Train the trees 33 | void Train(); 34 | 35 | // Save serialized trees into file 36 | void SaveTrees(std::string output_file); 37 | 38 | // Read trees from file 39 | void LoadTrees(std::string input_file); 40 | 41 | // Return the predicted x's label; optionally return the vote 42 | // (votes->size() will be num_labels_). 43 | int32_t Predict(const petuum::ml::AbstractFeature& x, 44 | std::vector* votes = 0) const; 45 | 46 | private: 47 | int32_t client_id_; 48 | 49 | int32_t thread_id_; 50 | 51 | int32_t num_threads_; 52 | 53 | int32_t num_trees_; 54 | 55 | int32_t num_labels_; 56 | 57 | bool save_trees_; 58 | 59 | std::vector trees_; 60 | 61 | DecisionTreeConfig tree_config_; 62 | 63 | std::vector serial_trees_; 64 | 65 | }; 66 | 67 | 68 | } // namespace tree 69 | -------------------------------------------------------------------------------- /app/rand_forest/src/utils.cpp: -------------------------------------------------------------------------------- 1 | // Author: Jiesi Zhao (jiesizhao0423@gmail.com), Wei Dai (wdai@cs.cmu.edu) 2 | // Date: 2014.11.5 3 | 4 | #include "utils.hpp" 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | namespace tree { 11 | 12 | // Compute Entropy of distribution (assuming \sum_i dist[i] = 1). 13 | float ComputeEntropy(const std::vector &dist) { 14 | float entropy = 0.0; 15 | for (auto iter = dist.begin(); iter != dist.end(); ++iter) { 16 | if (*iter) { 17 | entropy -= (*iter) * fastlog2(*iter); 18 | } 19 | } 20 | if (entropy < 1e-5) { 21 | entropy = 0.0; 22 | } 23 | return entropy; 24 | } 25 | 26 | // Normalize 27 | void Normalize(std::vector* count) { 28 | float sum = 0.; 29 | for (auto iter = count->begin(); iter != count->end(); ++iter) { 30 | sum += (*iter); 31 | } 32 | for (auto iter = count->begin(); iter != count->end(); ++iter) { 33 | (*iter) /= sum; 34 | } 35 | } 36 | 37 | } // namespace tree 38 | -------------------------------------------------------------------------------- /app/rand_forest/src/utils.hpp: -------------------------------------------------------------------------------- 1 | // Author: Jiesi Zhao (jiesizhao0423@gmail.com), Wei Dai (wdai@cs.cmu.edu) 2 | // Date: 2014.11.4 3 | 4 | 5 | #pragma once 6 | #include 7 | 8 | namespace tree { 9 | 10 | // Compute Entropy of distribution (assuming \sum_i dist[i] = 1). 11 | float ComputeEntropy(const std::vector &dist); 12 | 13 | // Normalize 14 | void Normalize(std::vector* count); 15 | 16 | } // namespace tree 17 | -------------------------------------------------------------------------------- /app/sparsecoding/script/kill.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os, sys 4 | 5 | if len(sys.argv) != 2: 6 | print "usage: %s " % sys.argv[0] 7 | sys.exit(1) 8 | 9 | host_file = sys.argv[1] 10 | prog_name = "sparsecoding_main" 11 | 12 | # Get host IPs 13 | with open(host_file, "r") as f: 14 | hostlines = f.read().splitlines() 15 | host_ips = [line.split()[1] for line in hostlines] 16 | 17 | ssh_cmd = ( 18 | "ssh " 19 | "-o StrictHostKeyChecking=no " 20 | "-o UserKnownHostsFile=/dev/null " 21 | "-o LogLevel=quiet " 22 | ) 23 | 24 | for ip in host_ips: 25 | cmd = ssh_cmd + ip + " killall -q " + prog_name 26 | os.system(cmd) 27 | print "Done killing" -------------------------------------------------------------------------------- /app/sparsecoding/script/launch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | from os.path import dirname, join 5 | import time 6 | 7 | hostfile_name = "machinefiles/localserver" 8 | 9 | app_dir = dirname(dirname(os.path.realpath(__file__))) 10 | proj_dir = dirname(dirname(app_dir)) 11 | 12 | hostfile = join(proj_dir, hostfile_name) 13 | 14 | ssh_cmd = ( 15 | "ssh " 16 | "-o StrictHostKeyChecking=no " 17 | "-o UserKnownHostsFile=/dev/null " 18 | ) 19 | 20 | # Get host IPs 21 | with open(hostfile, "r") as f: 22 | hostlines = f.read().splitlines() 23 | host_ips = [line.split()[1] for line in hostlines] 24 | 25 | for client_id, ip in enumerate(host_ips): 26 | cmd = ssh_cmd + ip + " " 27 | cmd += "\'python " + join(app_dir, "script/run_local.py") 28 | cmd += " %d %s\'" % (client_id, hostfile) 29 | cmd += " &" 30 | print cmd 31 | os.system(cmd) 32 | 33 | if client_id == 0: 34 | print "Waiting for first client to set up" 35 | time.sleep(2) -------------------------------------------------------------------------------- /app/sparsecoding/script/launch_on_yarn.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | from os.path import dirname 5 | from os.path import join 6 | import time 7 | 8 | app_dir = dirname(dirname(os.path.realpath(__file__))) 9 | proj_dir = dirname(dirname(app_dir)) 10 | 11 | params = { 12 | "jar": join(proj_dir, "src/yarn/build/libs/yarn-0.5.jar") 13 | , "launch_script_path": join(app_dir, "script", "run_local.py") 14 | , "container_memory": 500 15 | , "container_vcores": 1 16 | , "master_memory": 350 17 | , "priority": 10 18 | , "num_nodes": 1 19 | } 20 | 21 | cmd = "hadoop jar " 22 | cmd += join(proj_dir, "src/yarn/build/libs/yarn-0.5.jar") + " " 23 | cmd += "cn.edu.buaa.act.petuumOnYarn.Client" 24 | cmd += "".join([" --%s %s" % (k,v) for k,v in params.items()]) 25 | print cmd 26 | os.system(cmd) -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/Array: -------------------------------------------------------------------------------- 1 | #ifndef EIGEN_ARRAY_MODULE_H 2 | #define EIGEN_ARRAY_MODULE_H 3 | 4 | // include Core first to handle Eigen2 support macros 5 | #include "Core" 6 | 7 | #ifndef EIGEN2_SUPPORT 8 | #error The Eigen/Array header does no longer exist in Eigen3. All that functionality has moved to Eigen/Core. 9 | #endif 10 | 11 | #endif // EIGEN_ARRAY_MODULE_H 12 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | include(RegexUtils) 2 | test_escape_string_as_regex() 3 | 4 | file(GLOB Eigen_directory_files "*") 5 | 6 | escape_string_as_regex(ESCAPED_CMAKE_CURRENT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}") 7 | 8 | foreach(f ${Eigen_directory_files}) 9 | if(NOT f MATCHES "\\.txt" AND NOT f MATCHES "${ESCAPED_CMAKE_CURRENT_SOURCE_DIR}/[.].+" AND NOT f MATCHES "${ESCAPED_CMAKE_CURRENT_SOURCE_DIR}/src") 10 | list(APPEND Eigen_directory_files_to_install ${f}) 11 | endif() 12 | endforeach(f ${Eigen_directory_files}) 13 | 14 | install(FILES 15 | ${Eigen_directory_files_to_install} 16 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen COMPONENT Devel 17 | ) 18 | 19 | add_subdirectory(src) 20 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/Cholesky: -------------------------------------------------------------------------------- 1 | #ifndef EIGEN_CHOLESKY_MODULE_H 2 | #define EIGEN_CHOLESKY_MODULE_H 3 | 4 | #include "Core" 5 | 6 | #include "src/Core/util/DisableStupidWarnings.h" 7 | 8 | /** \defgroup Cholesky_Module Cholesky module 9 | * 10 | * 11 | * 12 | * This module provides two variants of the Cholesky decomposition for selfadjoint (hermitian) matrices. 13 | * Those decompositions are accessible via the following MatrixBase methods: 14 | * - MatrixBase::llt(), 15 | * - MatrixBase::ldlt() 16 | * 17 | * \code 18 | * #include 19 | * \endcode 20 | */ 21 | 22 | #include "src/misc/Solve.h" 23 | #include "src/Cholesky/LLT.h" 24 | #include "src/Cholesky/LDLT.h" 25 | #ifdef EIGEN_USE_LAPACKE 26 | #include "src/Cholesky/LLT_MKL.h" 27 | #endif 28 | 29 | #include "src/Core/util/ReenableStupidWarnings.h" 30 | 31 | #endif // EIGEN_CHOLESKY_MODULE_H 32 | /* vim: set filetype=cpp et sw=2 ts=2 ai: */ 33 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/Dense: -------------------------------------------------------------------------------- 1 | #include "Core" 2 | #include "LU" 3 | #include "Cholesky" 4 | #include "QR" 5 | #include "SVD" 6 | #include "Geometry" 7 | #include "Eigenvalues" 8 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/Eigen: -------------------------------------------------------------------------------- 1 | #include "Dense" 2 | //#include "Sparse" 3 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/Eigenvalues: -------------------------------------------------------------------------------- 1 | #ifndef EIGEN_EIGENVALUES_MODULE_H 2 | #define EIGEN_EIGENVALUES_MODULE_H 3 | 4 | #include "Core" 5 | 6 | #include "src/Core/util/DisableStupidWarnings.h" 7 | 8 | #include "Cholesky" 9 | #include "Jacobi" 10 | #include "Householder" 11 | #include "LU" 12 | #include "Geometry" 13 | 14 | /** \defgroup Eigenvalues_Module Eigenvalues module 15 | * 16 | * 17 | * 18 | * This module mainly provides various eigenvalue solvers. 19 | * This module also provides some MatrixBase methods, including: 20 | * - MatrixBase::eigenvalues(), 21 | * - MatrixBase::operatorNorm() 22 | * 23 | * \code 24 | * #include 25 | * \endcode 26 | */ 27 | 28 | #include "src/Eigenvalues/Tridiagonalization.h" 29 | #include "src/Eigenvalues/RealSchur.h" 30 | #include "src/Eigenvalues/EigenSolver.h" 31 | #include "src/Eigenvalues/SelfAdjointEigenSolver.h" 32 | #include "src/Eigenvalues/GeneralizedSelfAdjointEigenSolver.h" 33 | #include "src/Eigenvalues/HessenbergDecomposition.h" 34 | #include "src/Eigenvalues/ComplexSchur.h" 35 | #include "src/Eigenvalues/ComplexEigenSolver.h" 36 | #include "src/Eigenvalues/RealQZ.h" 37 | #include "src/Eigenvalues/GeneralizedEigenSolver.h" 38 | #include "src/Eigenvalues/MatrixBaseEigenvalues.h" 39 | #ifdef EIGEN_USE_LAPACKE 40 | #include "src/Eigenvalues/RealSchur_MKL.h" 41 | #include "src/Eigenvalues/ComplexSchur_MKL.h" 42 | #include "src/Eigenvalues/SelfAdjointEigenSolver_MKL.h" 43 | #endif 44 | 45 | #include "src/Core/util/ReenableStupidWarnings.h" 46 | 47 | #endif // EIGEN_EIGENVALUES_MODULE_H 48 | /* vim: set filetype=cpp et sw=2 ts=2 ai: */ 49 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/Householder: -------------------------------------------------------------------------------- 1 | #ifndef EIGEN_HOUSEHOLDER_MODULE_H 2 | #define EIGEN_HOUSEHOLDER_MODULE_H 3 | 4 | #include "Core" 5 | 6 | #include "src/Core/util/DisableStupidWarnings.h" 7 | 8 | /** \defgroup Householder_Module Householder module 9 | * This module provides Householder transformations. 10 | * 11 | * \code 12 | * #include 13 | * \endcode 14 | */ 15 | 16 | #include "src/Householder/Householder.h" 17 | #include "src/Householder/HouseholderSequence.h" 18 | #include "src/Householder/BlockHouseholder.h" 19 | 20 | #include "src/Core/util/ReenableStupidWarnings.h" 21 | 22 | #endif // EIGEN_HOUSEHOLDER_MODULE_H 23 | /* vim: set filetype=cpp et sw=2 ts=2 ai: */ 24 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/Jacobi: -------------------------------------------------------------------------------- 1 | #ifndef EIGEN_JACOBI_MODULE_H 2 | #define EIGEN_JACOBI_MODULE_H 3 | 4 | #include "Core" 5 | 6 | #include "src/Core/util/DisableStupidWarnings.h" 7 | 8 | /** \defgroup Jacobi_Module Jacobi module 9 | * This module provides Jacobi and Givens rotations. 10 | * 11 | * \code 12 | * #include 13 | * \endcode 14 | * 15 | * In addition to listed classes, it defines the two following MatrixBase methods to apply a Jacobi or Givens rotation: 16 | * - MatrixBase::applyOnTheLeft() 17 | * - MatrixBase::applyOnTheRight(). 18 | */ 19 | 20 | #include "src/Jacobi/Jacobi.h" 21 | 22 | #include "src/Core/util/ReenableStupidWarnings.h" 23 | 24 | #endif // EIGEN_JACOBI_MODULE_H 25 | /* vim: set filetype=cpp et sw=2 ts=2 ai: */ 26 | 27 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/LU: -------------------------------------------------------------------------------- 1 | #ifndef EIGEN_LU_MODULE_H 2 | #define EIGEN_LU_MODULE_H 3 | 4 | #include "Core" 5 | 6 | #include "src/Core/util/DisableStupidWarnings.h" 7 | 8 | /** \defgroup LU_Module LU module 9 | * This module includes %LU decomposition and related notions such as matrix inversion and determinant. 10 | * This module defines the following MatrixBase methods: 11 | * - MatrixBase::inverse() 12 | * - MatrixBase::determinant() 13 | * 14 | * \code 15 | * #include 16 | * \endcode 17 | */ 18 | 19 | #include "src/misc/Solve.h" 20 | #include "src/misc/Kernel.h" 21 | #include "src/misc/Image.h" 22 | #include "src/LU/FullPivLU.h" 23 | #include "src/LU/PartialPivLU.h" 24 | #ifdef EIGEN_USE_LAPACKE 25 | #include "src/LU/PartialPivLU_MKL.h" 26 | #endif 27 | #include "src/LU/Determinant.h" 28 | #include "src/LU/Inverse.h" 29 | 30 | #if defined EIGEN_VECTORIZE_SSE 31 | #include "src/LU/arch/Inverse_SSE.h" 32 | #endif 33 | 34 | #ifdef EIGEN2_SUPPORT 35 | #include "src/Eigen2Support/LU.h" 36 | #endif 37 | 38 | #include "src/Core/util/ReenableStupidWarnings.h" 39 | 40 | #endif // EIGEN_LU_MODULE_H 41 | /* vim: set filetype=cpp et sw=2 ts=2 ai: */ 42 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/LeastSquares: -------------------------------------------------------------------------------- 1 | #ifndef EIGEN_REGRESSION_MODULE_H 2 | #define EIGEN_REGRESSION_MODULE_H 3 | 4 | #ifndef EIGEN2_SUPPORT 5 | #error LeastSquares is only available in Eigen2 support mode (define EIGEN2_SUPPORT) 6 | #endif 7 | 8 | // exclude from normal eigen3-only documentation 9 | #ifdef EIGEN2_SUPPORT 10 | 11 | #include "Core" 12 | 13 | #include "src/Core/util/DisableStupidWarnings.h" 14 | 15 | #include "Eigenvalues" 16 | #include "Geometry" 17 | 18 | /** \defgroup LeastSquares_Module LeastSquares module 19 | * This module provides linear regression and related features. 20 | * 21 | * \code 22 | * #include 23 | * \endcode 24 | */ 25 | 26 | #include "src/Eigen2Support/LeastSquares.h" 27 | 28 | #include "src/Core/util/ReenableStupidWarnings.h" 29 | 30 | #endif // EIGEN2_SUPPORT 31 | 32 | #endif // EIGEN_REGRESSION_MODULE_H 33 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/MetisSupport: -------------------------------------------------------------------------------- 1 | #ifndef EIGEN_METISSUPPORT_MODULE_H 2 | #define EIGEN_METISSUPPORT_MODULE_H 3 | 4 | #include "SparseCore" 5 | 6 | #include "src/Core/util/DisableStupidWarnings.h" 7 | 8 | extern "C" { 9 | #include 10 | } 11 | 12 | 13 | /** \ingroup Support_modules 14 | * \defgroup MetisSupport_Module MetisSupport module 15 | * 16 | * \code 17 | * #include 18 | * \endcode 19 | * This module defines an interface to the METIS reordering package (http://glaros.dtc.umn.edu/gkhome/views/metis). 20 | * It can be used just as any other built-in method as explained in \link OrderingMethods_Module here. \endlink 21 | */ 22 | 23 | 24 | #include "src/MetisSupport/MetisSupport.h" 25 | 26 | #include "src/Core/util/ReenableStupidWarnings.h" 27 | 28 | #endif // EIGEN_METISSUPPORT_MODULE_H 29 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/PaStiXSupport: -------------------------------------------------------------------------------- 1 | #ifndef EIGEN_PASTIXSUPPORT_MODULE_H 2 | #define EIGEN_PASTIXSUPPORT_MODULE_H 3 | 4 | #include "SparseCore" 5 | 6 | #include "src/Core/util/DisableStupidWarnings.h" 7 | 8 | #include 9 | extern "C" { 10 | #include 11 | #include 12 | } 13 | 14 | #ifdef complex 15 | #undef complex 16 | #endif 17 | 18 | /** \ingroup Support_modules 19 | * \defgroup PaStiXSupport_Module PaStiXSupport module 20 | * 21 | * This module provides an interface to the PaSTiX library. 22 | * PaSTiX is a general \b supernodal, \b parallel and \b opensource sparse solver. 23 | * It provides the two following main factorization classes: 24 | * - class PastixLLT : a supernodal, parallel LLt Cholesky factorization. 25 | * - class PastixLDLT: a supernodal, parallel LDLt Cholesky factorization. 26 | * - class PastixLU : a supernodal, parallel LU factorization (optimized for a symmetric pattern). 27 | * 28 | * \code 29 | * #include 30 | * \endcode 31 | * 32 | * In order to use this module, the PaSTiX headers must be accessible from the include paths, and your binary must be linked to the PaSTiX library and its dependencies. 33 | * The dependencies depend on how PaSTiX has been compiled. 34 | * For a cmake based project, you can use our FindPaSTiX.cmake module to help you in this task. 35 | * 36 | */ 37 | 38 | #include "src/misc/Solve.h" 39 | #include "src/misc/SparseSolve.h" 40 | 41 | #include "src/PaStiXSupport/PaStiXSupport.h" 42 | 43 | 44 | #include "src/Core/util/ReenableStupidWarnings.h" 45 | 46 | #endif // EIGEN_PASTIXSUPPORT_MODULE_H 47 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/PardisoSupport: -------------------------------------------------------------------------------- 1 | #ifndef EIGEN_PARDISOSUPPORT_MODULE_H 2 | #define EIGEN_PARDISOSUPPORT_MODULE_H 3 | 4 | #include "SparseCore" 5 | 6 | #include "src/Core/util/DisableStupidWarnings.h" 7 | 8 | #include 9 | 10 | #include 11 | 12 | /** \ingroup Support_modules 13 | * \defgroup PardisoSupport_Module PardisoSupport module 14 | * 15 | * This module brings support for the Intel(R) MKL PARDISO direct sparse solvers. 16 | * 17 | * \code 18 | * #include 19 | * \endcode 20 | * 21 | * In order to use this module, the MKL headers must be accessible from the include paths, and your binary must be linked to the MKL library and its dependencies. 22 | * See this \ref TopicUsingIntelMKL "page" for more information on MKL-Eigen integration. 23 | * 24 | */ 25 | 26 | #include "src/PardisoSupport/PardisoSupport.h" 27 | 28 | #include "src/Core/util/ReenableStupidWarnings.h" 29 | 30 | #endif // EIGEN_PARDISOSUPPORT_MODULE_H 31 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/QR: -------------------------------------------------------------------------------- 1 | #ifndef EIGEN_QR_MODULE_H 2 | #define EIGEN_QR_MODULE_H 3 | 4 | #include "Core" 5 | 6 | #include "src/Core/util/DisableStupidWarnings.h" 7 | 8 | #include "Cholesky" 9 | #include "Jacobi" 10 | #include "Householder" 11 | 12 | /** \defgroup QR_Module QR module 13 | * 14 | * 15 | * 16 | * This module provides various QR decompositions 17 | * This module also provides some MatrixBase methods, including: 18 | * - MatrixBase::qr(), 19 | * 20 | * \code 21 | * #include 22 | * \endcode 23 | */ 24 | 25 | #include "src/misc/Solve.h" 26 | #include "src/QR/HouseholderQR.h" 27 | #include "src/QR/FullPivHouseholderQR.h" 28 | #include "src/QR/ColPivHouseholderQR.h" 29 | #ifdef EIGEN_USE_LAPACKE 30 | #include "src/QR/HouseholderQR_MKL.h" 31 | #include "src/QR/ColPivHouseholderQR_MKL.h" 32 | #endif 33 | 34 | #ifdef EIGEN2_SUPPORT 35 | #include "src/Eigen2Support/QR.h" 36 | #endif 37 | 38 | #include "src/Core/util/ReenableStupidWarnings.h" 39 | 40 | #ifdef EIGEN2_SUPPORT 41 | #include "Eigenvalues" 42 | #endif 43 | 44 | #endif // EIGEN_QR_MODULE_H 45 | /* vim: set filetype=cpp et sw=2 ts=2 ai: */ 46 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/QtAlignedMalloc: -------------------------------------------------------------------------------- 1 | 2 | #ifndef EIGEN_QTMALLOC_MODULE_H 3 | #define EIGEN_QTMALLOC_MODULE_H 4 | 5 | #include "Core" 6 | 7 | #if (!EIGEN_MALLOC_ALREADY_ALIGNED) 8 | 9 | #include "src/Core/util/DisableStupidWarnings.h" 10 | 11 | void *qMalloc(size_t size) 12 | { 13 | return Eigen::internal::aligned_malloc(size); 14 | } 15 | 16 | void qFree(void *ptr) 17 | { 18 | Eigen::internal::aligned_free(ptr); 19 | } 20 | 21 | void *qRealloc(void *ptr, size_t size) 22 | { 23 | void* newPtr = Eigen::internal::aligned_malloc(size); 24 | memcpy(newPtr, ptr, size); 25 | Eigen::internal::aligned_free(ptr); 26 | return newPtr; 27 | } 28 | 29 | #include "src/Core/util/ReenableStupidWarnings.h" 30 | 31 | #endif 32 | 33 | #endif // EIGEN_QTMALLOC_MODULE_H 34 | /* vim: set filetype=cpp et sw=2 ts=2 ai: */ 35 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/SPQRSupport: -------------------------------------------------------------------------------- 1 | #ifndef EIGEN_SPQRSUPPORT_MODULE_H 2 | #define EIGEN_SPQRSUPPORT_MODULE_H 3 | 4 | #include "SparseCore" 5 | 6 | #include "src/Core/util/DisableStupidWarnings.h" 7 | 8 | #include "SuiteSparseQR.hpp" 9 | 10 | /** \ingroup Support_modules 11 | * \defgroup SPQRSupport_Module SuiteSparseQR module 12 | * 13 | * This module provides an interface to the SPQR library, which is part of the suitesparse package. 14 | * 15 | * \code 16 | * #include 17 | * \endcode 18 | * 19 | * In order to use this module, the SPQR headers must be accessible from the include paths, and your binary must be linked to the SPQR library and its dependencies (Cholmod, AMD, COLAMD,...). 20 | * For a cmake based project, you can use our FindSPQR.cmake and FindCholmod.Cmake modules 21 | * 22 | */ 23 | 24 | #include "src/misc/Solve.h" 25 | #include "src/misc/SparseSolve.h" 26 | #include "src/CholmodSupport/CholmodSupport.h" 27 | #include "src/SPQRSupport/SuiteSparseQRSupport.h" 28 | 29 | #endif 30 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/SVD: -------------------------------------------------------------------------------- 1 | #ifndef EIGEN_SVD_MODULE_H 2 | #define EIGEN_SVD_MODULE_H 3 | 4 | #include "QR" 5 | #include "Householder" 6 | #include "Jacobi" 7 | 8 | #include "src/Core/util/DisableStupidWarnings.h" 9 | 10 | /** \defgroup SVD_Module SVD module 11 | * 12 | * 13 | * 14 | * This module provides SVD decomposition for matrices (both real and complex). 15 | * This decomposition is accessible via the following MatrixBase method: 16 | * - MatrixBase::jacobiSvd() 17 | * 18 | * \code 19 | * #include 20 | * \endcode 21 | */ 22 | 23 | #include "src/misc/Solve.h" 24 | #include "src/SVD/JacobiSVD.h" 25 | #if defined(EIGEN_USE_LAPACKE) && !defined(EIGEN_USE_LAPACKE_STRICT) 26 | #include "src/SVD/JacobiSVD_MKL.h" 27 | #endif 28 | #include "src/SVD/UpperBidiagonalization.h" 29 | 30 | #ifdef EIGEN2_SUPPORT 31 | #include "src/Eigen2Support/SVD.h" 32 | #endif 33 | 34 | #include "src/Core/util/ReenableStupidWarnings.h" 35 | 36 | #endif // EIGEN_SVD_MODULE_H 37 | /* vim: set filetype=cpp et sw=2 ts=2 ai: */ 38 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/Sparse: -------------------------------------------------------------------------------- 1 | #ifndef EIGEN_SPARSE_MODULE_H 2 | #define EIGEN_SPARSE_MODULE_H 3 | 4 | /** \defgroup Sparse_Module Sparse meta-module 5 | * 6 | * Meta-module including all related modules: 7 | * - \ref SparseCore_Module 8 | * - \ref OrderingMethods_Module 9 | * - \ref SparseCholesky_Module 10 | * - \ref SparseLU_Module 11 | * - \ref SparseQR_Module 12 | * - \ref IterativeLinearSolvers_Module 13 | * 14 | * \code 15 | * #include 16 | * \endcode 17 | */ 18 | 19 | #include "SparseCore" 20 | #include "OrderingMethods" 21 | #include "SparseCholesky" 22 | #include "SparseLU" 23 | #include "SparseQR" 24 | #include "IterativeLinearSolvers" 25 | 26 | #endif // EIGEN_SPARSE_MODULE_H 27 | 28 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/SparseCholesky: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2008-2013 Gael Guennebaud 5 | // 6 | // This Source Code Form is subject to the terms of the Mozilla 7 | // Public License v. 2.0. If a copy of the MPL was not distributed 8 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | 10 | #ifndef EIGEN_SPARSECHOLESKY_MODULE_H 11 | #define EIGEN_SPARSECHOLESKY_MODULE_H 12 | 13 | #include "SparseCore" 14 | #include "OrderingMethods" 15 | 16 | #include "src/Core/util/DisableStupidWarnings.h" 17 | 18 | /** 19 | * \defgroup SparseCholesky_Module SparseCholesky module 20 | * 21 | * This module currently provides two variants of the direct sparse Cholesky decomposition for selfadjoint (hermitian) matrices. 22 | * Those decompositions are accessible via the following classes: 23 | * - SimplicialLLt, 24 | * - SimplicialLDLt 25 | * 26 | * Such problems can also be solved using the ConjugateGradient solver from the IterativeLinearSolvers module. 27 | * 28 | * \code 29 | * #include 30 | * \endcode 31 | */ 32 | 33 | #ifdef EIGEN_MPL2_ONLY 34 | #error The SparseCholesky module has nothing to offer in MPL2 only mode 35 | #endif 36 | 37 | #include "src/misc/Solve.h" 38 | #include "src/misc/SparseSolve.h" 39 | #include "src/SparseCholesky/SimplicialCholesky.h" 40 | 41 | #ifndef EIGEN_MPL2_ONLY 42 | #include "src/SparseCholesky/SimplicialCholesky_impl.h" 43 | #endif 44 | 45 | #include "src/Core/util/ReenableStupidWarnings.h" 46 | 47 | #endif // EIGEN_SPARSECHOLESKY_MODULE_H 48 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/SparseQR: -------------------------------------------------------------------------------- 1 | #ifndef EIGEN_SPARSEQR_MODULE_H 2 | #define EIGEN_SPARSEQR_MODULE_H 3 | 4 | #include "SparseCore" 5 | #include "OrderingMethods" 6 | #include "src/Core/util/DisableStupidWarnings.h" 7 | 8 | /** \defgroup SparseQR_Module SparseQR module 9 | * \brief Provides QR decomposition for sparse matrices 10 | * 11 | * This module provides a simplicial version of the left-looking Sparse QR decomposition. 12 | * The columns of the input matrix should be reordered to limit the fill-in during the 13 | * decomposition. Built-in methods (COLAMD, AMD) or external methods (METIS) can be used to this end. 14 | * See the \link OrderingMethods_Module OrderingMethods\endlink module for the list 15 | * of built-in and external ordering methods. 16 | * 17 | * \code 18 | * #include 19 | * \endcode 20 | * 21 | * 22 | */ 23 | 24 | #include "src/misc/Solve.h" 25 | #include "src/misc/SparseSolve.h" 26 | 27 | #include "OrderingMethods" 28 | #include "src/SparseCore/SparseColEtree.h" 29 | #include "src/SparseQR/SparseQR.h" 30 | 31 | #include "src/Core/util/ReenableStupidWarnings.h" 32 | 33 | #endif 34 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/StdDeque: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2009 Gael Guennebaud 5 | // Copyright (C) 2009 Hauke Heibel 6 | // 7 | // This Source Code Form is subject to the terms of the Mozilla 8 | // Public License v. 2.0. If a copy of the MPL was not distributed 9 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 10 | 11 | #ifndef EIGEN_STDDEQUE_MODULE_H 12 | #define EIGEN_STDDEQUE_MODULE_H 13 | 14 | #include "Core" 15 | #include 16 | 17 | #if (defined(_MSC_VER) && defined(_WIN64)) /* MSVC auto aligns in 64 bit builds */ 18 | 19 | #define EIGEN_DEFINE_STL_DEQUE_SPECIALIZATION(...) 20 | 21 | #else 22 | 23 | #include "src/StlSupport/StdDeque.h" 24 | 25 | #endif 26 | 27 | #endif // EIGEN_STDDEQUE_MODULE_H 28 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/StdList: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2009 Hauke Heibel 5 | // 6 | // This Source Code Form is subject to the terms of the Mozilla 7 | // Public License v. 2.0. If a copy of the MPL was not distributed 8 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | 10 | #ifndef EIGEN_STDLIST_MODULE_H 11 | #define EIGEN_STDLIST_MODULE_H 12 | 13 | #include "Core" 14 | #include 15 | 16 | #if (defined(_MSC_VER) && defined(_WIN64)) /* MSVC auto aligns in 64 bit builds */ 17 | 18 | #define EIGEN_DEFINE_STL_LIST_SPECIALIZATION(...) 19 | 20 | #else 21 | 22 | #include "src/StlSupport/StdList.h" 23 | 24 | #endif 25 | 26 | #endif // EIGEN_STDLIST_MODULE_H 27 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/StdVector: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2009 Gael Guennebaud 5 | // Copyright (C) 2009 Hauke Heibel 6 | // 7 | // This Source Code Form is subject to the terms of the Mozilla 8 | // Public License v. 2.0. If a copy of the MPL was not distributed 9 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 10 | 11 | #ifndef EIGEN_STDVECTOR_MODULE_H 12 | #define EIGEN_STDVECTOR_MODULE_H 13 | 14 | #include "Core" 15 | #include 16 | 17 | #if (defined(_MSC_VER) && defined(_WIN64)) /* MSVC auto aligns in 64 bit builds */ 18 | 19 | #define EIGEN_DEFINE_STL_VECTOR_SPECIALIZATION(...) 20 | 21 | #else 22 | 23 | #include "src/StlSupport/StdVector.h" 24 | 25 | #endif 26 | 27 | #endif // EIGEN_STDVECTOR_MODULE_H 28 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/UmfPackSupport: -------------------------------------------------------------------------------- 1 | #ifndef EIGEN_UMFPACKSUPPORT_MODULE_H 2 | #define EIGEN_UMFPACKSUPPORT_MODULE_H 3 | 4 | #include "SparseCore" 5 | 6 | #include "src/Core/util/DisableStupidWarnings.h" 7 | 8 | extern "C" { 9 | #include 10 | } 11 | 12 | /** \ingroup Support_modules 13 | * \defgroup UmfPackSupport_Module UmfPackSupport module 14 | * 15 | * This module provides an interface to the UmfPack library which is part of the suitesparse package. 16 | * It provides the following factorization class: 17 | * - class UmfPackLU: a multifrontal sequential LU factorization. 18 | * 19 | * \code 20 | * #include 21 | * \endcode 22 | * 23 | * In order to use this module, the umfpack headers must be accessible from the include paths, and your binary must be linked to the umfpack library and its dependencies. 24 | * The dependencies depend on how umfpack has been compiled. 25 | * For a cmake based project, you can use our FindUmfPack.cmake module to help you in this task. 26 | * 27 | */ 28 | 29 | #include "src/misc/Solve.h" 30 | #include "src/misc/SparseSolve.h" 31 | 32 | #include "src/UmfPackSupport/UmfPackSupport.h" 33 | 34 | #include "src/Core/util/ReenableStupidWarnings.h" 35 | 36 | #endif // EIGEN_UMFPACKSUPPORT_MODULE_H 37 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB Eigen_src_subdirectories "*") 2 | escape_string_as_regex(ESCAPED_CMAKE_CURRENT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}") 3 | foreach(f ${Eigen_src_subdirectories}) 4 | if(NOT f MATCHES "\\.txt" AND NOT f MATCHES "${ESCAPED_CMAKE_CURRENT_SOURCE_DIR}/[.].+" ) 5 | add_subdirectory(${f}) 6 | endif() 7 | endforeach() 8 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/Cholesky/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_Cholesky_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_Cholesky_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Cholesky COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/CholmodSupport/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_CholmodSupport_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_CholmodSupport_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/CholmodSupport COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/Core/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_Core_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_Core_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Core COMPONENT Devel 6 | ) 7 | 8 | ADD_SUBDIRECTORY(products) 9 | ADD_SUBDIRECTORY(util) 10 | ADD_SUBDIRECTORY(arch) 11 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/Core/arch/AltiVec/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_Core_arch_AltiVec_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_Core_arch_AltiVec_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Core/arch/AltiVec COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/Core/arch/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | ADD_SUBDIRECTORY(SSE) 2 | ADD_SUBDIRECTORY(AltiVec) 3 | ADD_SUBDIRECTORY(NEON) 4 | ADD_SUBDIRECTORY(Default) 5 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/Core/arch/Default/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_Core_arch_Default_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_Core_arch_Default_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Core/arch/Default COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/Core/arch/NEON/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_Core_arch_NEON_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_Core_arch_NEON_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Core/arch/NEON COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/Core/arch/SSE/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_Core_arch_SSE_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_Core_arch_SSE_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Core/arch/SSE COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/Core/products/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_Core_Product_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_Core_Product_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Core/products COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/Core/util/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_Core_util_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_Core_util_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Core/util COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/Core/util/NonMPL2.h: -------------------------------------------------------------------------------- 1 | #ifdef EIGEN_MPL2_ONLY 2 | #error Including non-MPL2 code in EIGEN_MPL2_ONLY mode 3 | #endif 4 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/Core/util/ReenableStupidWarnings.h: -------------------------------------------------------------------------------- 1 | #ifdef EIGEN_WARNINGS_DISABLED 2 | #undef EIGEN_WARNINGS_DISABLED 3 | 4 | #ifndef EIGEN_PERMANENTLY_DISABLE_STUPID_WARNINGS 5 | #ifdef _MSC_VER 6 | #pragma warning( pop ) 7 | #elif defined __INTEL_COMPILER 8 | #pragma warning pop 9 | #elif defined __clang__ 10 | #pragma clang diagnostic pop 11 | #endif 12 | #endif 13 | 14 | #endif // EIGEN_WARNINGS_DISABLED 15 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/Eigen2Support/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_Eigen2Support_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_Eigen2Support_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Eigen2Support COMPONENT Devel 6 | ) 7 | 8 | ADD_SUBDIRECTORY(Geometry) -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/Eigen2Support/Geometry/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_Eigen2Support_Geometry_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_Eigen2Support_Geometry_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Eigen2Support/Geometry 6 | ) 7 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/Eigen2Support/Macros.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2011 Benoit Jacob 5 | // 6 | // This Source Code Form is subject to the terms of the Mozilla 7 | // Public License v. 2.0. If a copy of the MPL was not distributed 8 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | 10 | #ifndef EIGEN2_MACROS_H 11 | #define EIGEN2_MACROS_H 12 | 13 | #define ei_assert eigen_assert 14 | #define ei_internal_assert eigen_internal_assert 15 | 16 | #define EIGEN_ALIGN_128 EIGEN_ALIGN16 17 | 18 | #define EIGEN_ARCH_WANTS_ALIGNMENT EIGEN_ALIGN_STATICALLY 19 | 20 | #endif // EIGEN2_MACROS_H 21 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/Eigenvalues/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_EIGENVALUES_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_EIGENVALUES_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Eigenvalues COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/Geometry/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_Geometry_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_Geometry_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Geometry COMPONENT Devel 6 | ) 7 | 8 | ADD_SUBDIRECTORY(arch) 9 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/Geometry/arch/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_Geometry_arch_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_Geometry_arch_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Geometry/arch COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/Householder/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_Householder_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_Householder_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Householder COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/IterativeLinearSolvers/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_IterativeLinearSolvers_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_IterativeLinearSolvers_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/IterativeLinearSolvers COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/Jacobi/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_Jacobi_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_Jacobi_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Jacobi COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/LU/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_LU_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_LU_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/LU COMPONENT Devel 6 | ) 7 | 8 | ADD_SUBDIRECTORY(arch) 9 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/LU/arch/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_LU_arch_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_LU_arch_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/LU/arch COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/MetisSupport/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_MetisSupport_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_MetisSupport_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/MetisSupport COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/OrderingMethods/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_OrderingMethods_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_OrderingMethods_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/OrderingMethods COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/PaStiXSupport/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_PastixSupport_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_PastixSupport_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/PaStiXSupport COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/PardisoSupport/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_PardisoSupport_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_PardisoSupport_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/PardisoSupport COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/QR/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_QR_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_QR_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/QR COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/SPQRSupport/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_SPQRSupport_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_SPQRSupport_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/SPQRSupport/ COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/SVD/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_SVD_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_SVD_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/SVD COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/SparseCholesky/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_SparseCholesky_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_SparseCholesky_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/SparseCholesky COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/SparseCore/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_SparseCore_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_SparseCore_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/SparseCore COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/SparseCore/SparseFuzzy.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2008 Gael Guennebaud 5 | // 6 | // This Source Code Form is subject to the terms of the Mozilla 7 | // Public License v. 2.0. If a copy of the MPL was not distributed 8 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | 10 | #ifndef EIGEN_SPARSE_FUZZY_H 11 | #define EIGEN_SPARSE_FUZZY_H 12 | 13 | // template 14 | // template 15 | // bool SparseMatrixBase::isApprox( 16 | // const OtherDerived& other, 17 | // typename NumTraits::Real prec 18 | // ) const 19 | // { 20 | // const typename internal::nested::type nested(derived()); 21 | // const typename internal::nested::type otherNested(other.derived()); 22 | // return (nested - otherNested).cwise().abs2().sum() 23 | // <= prec * prec * (std::min)(nested.cwise().abs2().sum(), otherNested.cwise().abs2().sum()); 24 | // } 25 | 26 | #endif // EIGEN_SPARSE_FUZZY_H 27 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/SparseLU/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_SparseLU_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_SparseLU_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/SparseLU COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/SparseQR/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_SparseQR_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_SparseQR_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/SparseQR/ COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/StlSupport/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_StlSupport_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_StlSupport_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/StlSupport COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/SuperLUSupport/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_SuperLUSupport_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_SuperLUSupport_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/SuperLUSupport COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/UmfPackSupport/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_UmfPackSupport_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_UmfPackSupport_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/UmfPackSupport COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/misc/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_misc_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_misc_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/misc COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /app/sparsecoding/src/util/Eigen/src/plugins/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FILE(GLOB Eigen_plugins_SRCS "*.h") 2 | 3 | INSTALL(FILES 4 | ${Eigen_plugins_SRCS} 5 | DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/plugins COMPONENT Devel 6 | ) 7 | -------------------------------------------------------------------------------- /docs/bosen_refman.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sailing-pmls/bosen/06cb58902d011fbea5f9428f10ce30e621492204/docs/bosen_refman.pdf -------------------------------------------------------------------------------- /machinefiles/localserver: -------------------------------------------------------------------------------- 1 | 0 127.0.0.1 9999 2 | -------------------------------------------------------------------------------- /src/ml/feature/abstract_datum.hpp: -------------------------------------------------------------------------------- 1 | // Author: Dai Wei (wdai@cs.cmu.edu) 2 | // Date: 2014.10.18 3 | 4 | #pragma once 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | namespace petuum { 11 | namespace ml { 12 | 13 | template 14 | class AbstractDatum { 15 | public: 16 | AbstractDatum(AbstractFeature* feature, LABEL label) : 17 | feature_(feature), label_(label) { } 18 | 19 | // Copy constructor transfer ownership of feature_. 20 | AbstractDatum(const AbstractDatum& other) : 21 | feature_(other.ReleaseFeature()), label_(other.label_) { } 22 | 23 | AbstractFeature* GetFeature() { 24 | CHECK(feature_) << "Accessing null feature."; 25 | return feature_.get(); 26 | } 27 | 28 | LABEL GetLabel() { 29 | return label_; 30 | } 31 | 32 | // Release ownership of feature. 33 | AbstractFeature* ReleaseFeature() { 34 | return feature_.release(); 35 | } 36 | 37 | private: 38 | // Takes ownership of feature_. 39 | std::unique_ptr > feature_; 40 | 41 | LABEL label_; 42 | }; 43 | 44 | } // namespace ml 45 | } // namespace petuum 46 | -------------------------------------------------------------------------------- /src/ml/include/ml.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include 10 | #include 11 | #include 12 | -------------------------------------------------------------------------------- /src/ml/util/fastapprox/fastapprox.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | -------------------------------------------------------------------------------- /src/ml/util/metafile_reader.hpp: -------------------------------------------------------------------------------- 1 | // Author: Dai Wei (wdai@cs.cmu.edu) 2 | // Date: 2014.10.18 3 | 4 | #pragma once 5 | 6 | #include 7 | #include 8 | 9 | namespace petuum { 10 | namespace ml { 11 | 12 | // Read meta file consisting of "name: value" on each line. 13 | class MetafileReader { 14 | public: 15 | MetafileReader(); 16 | MetafileReader(const std::string& metafile_path); 17 | void Init(const std::string& metafile_path); 18 | 19 | int get_int32(std::string key); 20 | double get_double(std::string key); 21 | bool get_bool(std::string key); 22 | std::string get_string(std::string key); 23 | 24 | private: 25 | std::unordered_map content_; 26 | 27 | std::string metafile_path_; 28 | 29 | }; 30 | 31 | } // namespace ml 32 | } // namespace petuum 33 | -------------------------------------------------------------------------------- /src/petuum.mk: -------------------------------------------------------------------------------- 1 | PS_DIR = $(SRC)/petuum_ps 2 | PS_SRC = $(shell find $(PS_DIR) -type f -name "*.cpp") 3 | PS_HEADERS = $(shell find $(PS_DIR) -type f -name "*.hpp") 4 | PS_OBJ = $(PS_SRC:.cpp=.o) 5 | IO_DIR = $(SRC)/io 6 | 7 | PS_COMMON_DIR = $(SRC)/petuum_ps_common 8 | PS_COMMON_SRC = $(shell find $(PS_COMMON_DIR) -type f -name "*.cpp") 9 | PS_COMMON_HEADERS = $(shell find $(PS_COMMON_DIR) -type f -name "*.hpp") 10 | PS_COMMON_OBJ = $(PS_COMMON_SRC:.cpp=.o) 11 | 12 | ML_DIR = $(SRC)/ml 13 | ML_SRC = $(shell find $(ML_DIR) $(IO_DIR) -type f -name "*.cpp") 14 | ML_HEADERS = $(shell find $(ML_DIR) -type f -name "*.hpp") 15 | ML_OBJ = $(ML_SRC:.cpp=.o) 16 | 17 | # ================== library ================== 18 | 19 | # PS Library 20 | PS_LIB = $(LIB)/libpetuum-ps.a 21 | 22 | $(PS_LIB): $(PS_OBJ) $(PS_COMMON_OBJ) path 23 | ar csrv $@ $(PS_OBJ) $(PS_COMMON_OBJ) 24 | 25 | $(PS_OBJ): %.o: %.cpp $(PS_HEADERS) $(PS_COMMON_HEADERS) 26 | $(CXX) $(CXXFLAGS) $(INCFLAGS) -c $< -o $@ 27 | 28 | $(PS_COMMON_OBJ): %.o: %.cpp $(PS_COMMON_HEADERS) 29 | $(CXX) $(CXXFLAGS) $(INCFLAGS) -c $< -o $@ 30 | 31 | ps_lib: $(PS_LIB) 32 | 33 | 34 | # ML Library 35 | ML_LIB = $(LIB)/libpetuum-ml.a 36 | 37 | $(ML_LIB): $(ML_OBJ) path 38 | ar csrv $@ $(ML_OBJ) 39 | 40 | $(ML_OBJ): %.o: %.cpp $(ML_HEADERS) 41 | $(CXX) $(CXXFLAGS) $(INCFLAGS) $(HDFS_INCFLAGS) $(HAS_HDFS) $(HDFS_LDFLAGS) -c $< -o $@ 42 | 43 | ml_lib: $(ML_LIB) 44 | 45 | 46 | clean_src: 47 | rm -rf $(IO_OBJ) 48 | 49 | .PHONY: ps_lib ml_lib 50 | -------------------------------------------------------------------------------- /src/petuum_ps/client/oplog_serializer.hpp: -------------------------------------------------------------------------------- 1 | // author: jinliang 2 | #pragma once 3 | 4 | #include 5 | 6 | namespace petuum { 7 | class OpLogSerializer : boost::noncopyable { 8 | public: 9 | OpLogSerializer() {} 10 | ~OpLogSerializer() {} 11 | 12 | size_t Init(const std::map &table_size_map) { 13 | num_tables_ = table_size_map.size(); 14 | if (num_tables_ == 0) 15 | return 0; 16 | 17 | // space for num of tables 18 | size_t total_size = sizeof(int32_t); 19 | for(auto iter = table_size_map.cbegin(); iter != table_size_map.cend(); 20 | iter++){ 21 | int32_t table_id = iter->first; 22 | size_t table_size = iter->second; 23 | offset_map_[table_id] = total_size; 24 | // next table is offset by 25 | // 1) the current table size and 26 | // 2) space for table id 27 | // 3) update size 28 | total_size += table_size + sizeof(int32_t) + sizeof(size_t); 29 | } 30 | return total_size; 31 | } 32 | 33 | // does not take ownership 34 | void AssignMem(void *mem) { 35 | mem_ = reinterpret_cast(mem); 36 | *(reinterpret_cast(mem_)) = num_tables_; 37 | } 38 | 39 | void *GetTablePtr(int32_t table_id) { 40 | auto iter = offset_map_.find(table_id); 41 | if (iter == offset_map_.end()) 42 | return 0; 43 | return mem_ + iter->second; 44 | } 45 | 46 | private: 47 | std::map offset_map_; 48 | uint8_t *mem_; 49 | int32_t num_tables_; 50 | }; 51 | 52 | } 53 | -------------------------------------------------------------------------------- /src/petuum_ps/client/ssp_client_row.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | // author: jinliang 4 | 5 | #include 6 | #include 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | namespace petuum { 16 | 17 | // ClientRow is a wrapper on user-defined ROW data structure (e.g., vector, 18 | // map) with additional features: 19 | // 20 | // 1. Reference Counting: number of references used by application. Note the 21 | // copy in storage itself does not contribute to the count 22 | // 2. Row Metadata 23 | // 24 | // ClientRow does not provide thread-safety in itself. The locks are 25 | // maintained in the storage and in (user-defined) ROW. 26 | class SSPClientRow : public ClientRow { 27 | public: 28 | // ClientRow takes ownership of row_data. 29 | SSPClientRow(int32_t clock, AbstractRow* row_data, bool use_ref_count): 30 | ClientRow(clock, row_data, use_ref_count), 31 | clock_(clock){ } 32 | 33 | void SetClock(int32_t clock) { 34 | std::unique_lock ulock(clock_mtx_); 35 | clock_ = clock; 36 | } 37 | 38 | int32_t GetClock() const { 39 | std::unique_lock ulock(clock_mtx_); 40 | return clock_; 41 | } 42 | 43 | private: // private members 44 | mutable std::mutex clock_mtx_; 45 | int32_t clock_; 46 | }; 47 | 48 | } // namespace petuum 49 | -------------------------------------------------------------------------------- /src/petuum_ps/consistency/ssp_aggr_consistency_controller.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | namespace petuum { 6 | 7 | class SSPAggrConsistencyController : public SSPPushConsistencyController { 8 | public: 9 | SSPAggrConsistencyController( 10 | const TableInfo& info, 11 | int32_t table_id, 12 | AbstractProcessStorage& process_storage, 13 | AbstractOpLog& oplog, 14 | const AbstractRow* sample_row, 15 | boost::thread_specific_ptr &thread_cache, 16 | TableOpLogIndex &oplog_index, 17 | int32_t row_oplog_type); 18 | 19 | // Return immediately. 20 | virtual void Inc(int32_t row_id, int32_t column_id, const void* delta); 21 | 22 | virtual void BatchInc(int32_t row_id, const int32_t* column_ids, 23 | const void* updates, int32_t num_updates); 24 | 25 | virtual void DenseBatchInc(int32_t row_id, const void *updates, 26 | int32_t index_st, int32_t num_updates); 27 | 28 | virtual void ThreadInc(int32_t row_id, int32_t column_id, const void* delta); 29 | 30 | // Increment column_ids.size() entries of a row. deltas points to an array. 31 | virtual void ThreadBatchInc(int32_t row_id, const int32_t* column_ids, 32 | const void* updates, int32_t num_updates); 33 | virtual void ThreadDenseBatchInc( 34 | int32_t row_id, const void *updates, int32_t index_st, 35 | int32_t num_updates); 36 | protected: 37 | void CheckAndFlushThreadCache(size_t thread_update_count); 38 | }; 39 | 40 | } // namespace petuum 41 | -------------------------------------------------------------------------------- /src/petuum_ps/consistency/ssp_aggr_value_consistency_controller.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | namespace petuum { 6 | 7 | class SSPAggrValueConsistencyController : public SSPAggrConsistencyController { 8 | public: 9 | SSPAggrValueConsistencyController( 10 | const TableInfo& info, 11 | int32_t table_id, 12 | AbstractProcessStorage& process_storage, 13 | AbstractOpLog& oplog, 14 | const AbstractRow* sample_row, 15 | boost::thread_specific_ptr &thread_cache, 16 | TableOpLogIndex &oplog_index, 17 | int32_t row_oplog_type); 18 | 19 | // Return immediately. 20 | virtual void Inc(int32_t row_id, int32_t column_id, const void* delta); 21 | 22 | virtual void BatchInc(int32_t row_id, const int32_t* column_ids, 23 | const void* updates, int32_t num_updates); 24 | 25 | virtual void DenseBatchInc(int32_t row_id, const void *updates, 26 | int32_t index_st, int32_t num_updates); 27 | 28 | }; 29 | } 30 | -------------------------------------------------------------------------------- /src/petuum_ps/consistency/ssp_push_append_only_consistency_controller.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | namespace petuum { 6 | 7 | class SSPPushAppendOnlyConsistencyController : 8 | public SSPPushConsistencyController { 9 | public: 10 | SSPPushAppendOnlyConsistencyController( 11 | const TableInfo& info, 12 | int32_t table_id, 13 | AbstractProcessStorage& process_storage, 14 | AbstractOpLog& oplog, 15 | const AbstractRow* sample_row, 16 | boost::thread_specific_ptr &thread_cache, 17 | TableOpLogIndex &oplog_index, 18 | int32_t row_oplog_type); 19 | 20 | virtual void Inc(int32_t row_id, int32_t column_id, const void* delta); 21 | 22 | virtual void BatchInc(int32_t row_id, const int32_t* column_ids, 23 | const void* updates, int32_t num_updates); 24 | 25 | virtual void DenseBatchInc(int32_t row_id, const void *updates, 26 | int32_t index_st, int32_t num_updates); 27 | 28 | virtual void ThreadInc(int32_t row_id, int32_t column_id, const void* delta); 29 | 30 | // Increment column_ids.size() entries of a row. deltas points to an array. 31 | virtual void ThreadBatchInc(int32_t row_id, const int32_t* column_ids, 32 | const void* updates, int32_t num_updates); 33 | 34 | virtual void ThreadDenseBatchInc( 35 | int32_t row_id, const void *updates, int32_t index_st, 36 | int32_t num_updates); 37 | 38 | virtual void FlushThreadCache(); 39 | virtual void Clock(); 40 | }; 41 | 42 | } // namespace petuum 43 | -------------------------------------------------------------------------------- /src/petuum_ps/consistency/ssp_push_consistency_controller.hpp: -------------------------------------------------------------------------------- 1 | // author: Jinliang 2 | #pragma once 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | namespace petuum { 12 | 13 | class SSPPushConsistencyController : public SSPConsistencyController { 14 | public: 15 | SSPPushConsistencyController( 16 | const TableInfo& info, 17 | int32_t table_id, 18 | AbstractProcessStorage& process_storage, 19 | AbstractOpLog& oplog, 20 | const AbstractRow* sample_row, 21 | boost::thread_specific_ptr &thread_cache, 22 | TableOpLogIndex &oplog_index, 23 | int32_t row_oplog_type); 24 | 25 | void GetAsyncForced(int32_t row_id); 26 | void GetAsync(int32_t row_id); 27 | void WaitPendingAsnycGet(); 28 | 29 | // Check freshness; make request and block if too stale or row_id not found 30 | // in storage. 31 | ClientRow *Get(int32_t row_id, RowAccessor* row_accessor); 32 | 33 | void ThreadGet(int32_t row_id, ThreadRowAccessor* row_accessor); 34 | 35 | private: 36 | static const size_t kMaxPendingAsyncGetCnt = 256; 37 | boost::thread_specific_ptr pending_async_get_cnt_; 38 | }; 39 | 40 | } // namespace petuum 41 | -------------------------------------------------------------------------------- /src/petuum_ps/oplog/create_row_oplog.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | namespace petuum { 11 | 12 | class CreateRowOpLog { 13 | public: 14 | typedef AbstractRowOpLog *(*CreateRowOpLogFunc)( 15 | size_t update_size, const AbstractRow *sample_row, 16 | size_t row_oplog_capacity); 17 | 18 | static AbstractRowOpLog *CreateDenseRowOpLog( 19 | size_t update_size, const AbstractRow *sample_row, 20 | size_t row_oplog_capacity); 21 | 22 | static AbstractRowOpLog *CreateSparseRowOpLog( 23 | size_t update_size, const AbstractRow *sample_row, 24 | size_t row_oplog_capacity); 25 | 26 | static AbstractRowOpLog *CreateSparseVectorRowOpLog( 27 | size_t update_size, const AbstractRow *sample_row, 28 | size_t row_oplog_capacity); 29 | 30 | static AbstractRowOpLog *CreateDenseMetaRowOpLog( 31 | size_t update_size, const AbstractRow *sample_row, 32 | size_t row_oplog_capacity); 33 | 34 | static AbstractRowOpLog *CreateSparseMetaRowOpLog( 35 | size_t update_size, const AbstractRow *sample_row, 36 | size_t row_oplog_capacity); 37 | 38 | static AbstractRowOpLog *CreateSparseVectorMetaRowOpLog( 39 | size_t update_size, const AbstractRow *sample_row, 40 | size_t row_oplog_capacity); 41 | 42 | }; 43 | } 44 | -------------------------------------------------------------------------------- /src/petuum_ps/oplog/oplog_index.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | namespace petuum { 14 | class PartitionOpLogIndex : boost::noncopyable { 15 | public: 16 | explicit PartitionOpLogIndex(size_t capacity); 17 | PartitionOpLogIndex(PartitionOpLogIndex && other); 18 | PartitionOpLogIndex & operator = (PartitionOpLogIndex && other) = delete; 19 | 20 | ~PartitionOpLogIndex(); 21 | void AddIndex(const std::unordered_set &oplog_index); 22 | cuckoohash_map *Reset(); 23 | size_t GetNumRowOpLogs(); 24 | private: 25 | size_t capacity_; 26 | SharedMutex smtx_; 27 | StripedLock locks_; 28 | cuckoohash_map *shared_oplog_index_; 29 | }; 30 | 31 | class TableOpLogIndex : boost::noncopyable{ 32 | public: 33 | explicit TableOpLogIndex(size_t capacity); 34 | void AddIndex(int32_t partition_num, 35 | const std::unordered_set &oplog_index); 36 | cuckoohash_map *ResetPartition(int32_t partition_num); 37 | size_t GetNumRowOpLogs(int32_t partition_num); 38 | private: 39 | std::vector partition_oplog_index_; 40 | }; 41 | } 42 | -------------------------------------------------------------------------------- /src/petuum_ps/oplog/row_oplog_meta.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | namespace petuum { 6 | 7 | class RowOpLogMeta { 8 | public: 9 | RowOpLogMeta(): 10 | clock_(-1), 11 | importance_(0) { } 12 | 13 | int32_t get_clock() const { 14 | return clock_; 15 | } 16 | 17 | void set_clock(int32_t clock) { 18 | if (clock_ == -1 || clock < clock_) 19 | clock_ = clock; 20 | } 21 | 22 | double get_importance() const { 23 | return importance_; 24 | } 25 | 26 | void set_importance(double importance) { 27 | importance_ = importance; 28 | } 29 | 30 | void accum_importance(double importance) { 31 | importance_ += importance; 32 | } 33 | 34 | private: 35 | int32_t clock_; 36 | double importance_; 37 | }; 38 | 39 | } 40 | -------------------------------------------------------------------------------- /src/petuum_ps/server/name_node.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | namespace petuum { 4 | NameNodeThread *NameNode::name_node_thread_; 5 | pthread_barrier_t NameNode::init_barrier_; 6 | 7 | void NameNode::Init() { 8 | pthread_barrier_init(&init_barrier_, NULL, 2); 9 | name_node_thread_ = new NameNodeThread(&init_barrier_); 10 | name_node_thread_->Start(); 11 | pthread_barrier_wait(&init_barrier_); 12 | } 13 | 14 | void NameNode::ShutDown() { 15 | name_node_thread_->ShutDown(); 16 | delete name_node_thread_; 17 | } 18 | 19 | } 20 | -------------------------------------------------------------------------------- /src/petuum_ps/server/name_node.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | namespace petuum { 7 | 8 | class NameNode { 9 | public: 10 | static void Init(); 11 | static void ShutDown(); 12 | private: 13 | static NameNodeThread *name_node_thread_; 14 | static pthread_barrier_t init_barrier_; 15 | }; 16 | 17 | } 18 | -------------------------------------------------------------------------------- /src/petuum_ps/server/server_thread_group.hpp: -------------------------------------------------------------------------------- 1 | // author: jinliang 2 | #pragma once 3 | 4 | #include 5 | 6 | namespace petuum { 7 | class ServerThreadGroup { 8 | public: 9 | ServerThreadGroup(int32_t server_id_st); 10 | ~ServerThreadGroup(); 11 | 12 | void Start(); 13 | void ShutDown(); 14 | private: 15 | std::vector server_thread_vec_; 16 | pthread_barrier_t init_barrier_; 17 | }; 18 | 19 | } 20 | -------------------------------------------------------------------------------- /src/petuum_ps/server/server_threads.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | namespace petuum { 4 | 5 | ServerThreadGroup *ServerThreads::server_thread_group_; 6 | 7 | void ServerThreads::Init(int32_t server_id_st) { 8 | server_thread_group_ = new ServerThreadGroup(server_id_st); 9 | server_thread_group_->Start(); 10 | } 11 | 12 | void ServerThreads::ShutDown() { 13 | server_thread_group_->ShutDown(); 14 | delete server_thread_group_; 15 | } 16 | 17 | } 18 | -------------------------------------------------------------------------------- /src/petuum_ps/server/server_threads.hpp: -------------------------------------------------------------------------------- 1 | // author: jinliang 2 | #pragma once 3 | 4 | #include 5 | 6 | namespace petuum { 7 | 8 | class ServerThreads { 9 | public: 10 | static void Init(int32_t server_id_st); 11 | static void ShutDown(); 12 | 13 | private: 14 | static ServerThreadGroup *server_thread_group_; 15 | }; 16 | 17 | } 18 | -------------------------------------------------------------------------------- /src/petuum_ps/server/ssp_aggr_server_thread.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | namespace petuum { 7 | void SSPAggrServerThread::SetWaitMsg() { 8 | WaitMsg_ = WaitMsgTimeOut; 9 | } 10 | 11 | long SSPAggrServerThread::ServerIdleWork() { 12 | if (row_send_milli_sec_ > 1) { 13 | double send_elapsed_milli = msg_send_timer_.elapsed() * kOneThousand; 14 | if (row_send_milli_sec_ > send_elapsed_milli + 1) 15 | return (row_send_milli_sec_ - send_elapsed_milli); 16 | } 17 | 18 | if (server_obj_.AccumedOpLogSinceLastPush()) { 19 | size_t sent_bytes 20 | = server_obj_.CreateSendServerPushRowMsgsPartial(SendServerPushRowMsg); 21 | row_send_milli_sec_ = TransTimeEstimate::EstimateTransMillisec(sent_bytes); 22 | 23 | msg_send_timer_.restart(); 24 | 25 | return row_send_milli_sec_; 26 | } 27 | 28 | return GlobalContext::get_server_idle_milli(); 29 | } 30 | 31 | long SSPAggrServerThread::ResetServerIdleMilli() { 32 | return GlobalContext::get_server_idle_milli(); 33 | } 34 | 35 | void SSPAggrServerThread::ServerPushRow(bool clock_changed) { 36 | STATS_SERVER_ACCUM_PUSH_ROW_BEGIN(); 37 | size_t sent_bytes 38 | = server_obj_.CreateSendServerPushRowMsgs(SendServerPushRowMsg); 39 | STATS_SERVER_ACCUM_PUSH_ROW_END(); 40 | 41 | row_send_milli_sec_ = TransTimeEstimate::EstimateTransMillisec(sent_bytes); 42 | msg_send_timer_.restart(); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/petuum_ps/server/ssp_aggr_server_thread.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | namespace petuum { 7 | 8 | class SSPAggrServerThread : public SSPPushServerThread { 9 | public: 10 | SSPAggrServerThread(int32_t my_id, pthread_barrier_t *init_barrier): 11 | SSPPushServerThread(my_id, init_barrier), 12 | row_send_milli_sec_(0) { } 13 | 14 | ~SSPAggrServerThread() { } 15 | 16 | protected: 17 | virtual void SetWaitMsg(); 18 | virtual long ServerIdleWork(); 19 | virtual long ResetServerIdleMilli(); 20 | virtual void ServerPushRow(bool clock_changed); 21 | 22 | HighResolutionTimer msg_send_timer_; 23 | double row_send_milli_sec_; 24 | }; 25 | 26 | } 27 | -------------------------------------------------------------------------------- /src/petuum_ps/server/ssp_push_server_thread.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | namespace petuum { 6 | 7 | class SSPPushServerThread : public ServerThread { 8 | public: 9 | SSPPushServerThread(int32_t my_id, pthread_barrier_t *init_barrier): 10 | ServerThread(my_id, init_barrier) { } 11 | 12 | ~SSPPushServerThread() { } 13 | protected: 14 | virtual void ServerPushRow(bool clock_changed); 15 | static void SendServerPushRowMsg (int32_t bg_id, ServerPushRowMsg *msg, 16 | bool last_msg, int32_t version, 17 | int32_t server_min_clock); 18 | 19 | virtual void RowSubscribe(ServerRow *server_row, int32_t client_id); 20 | 21 | virtual void SendOpLogAckMsg(int32_t bg_id, uint32_t version); 22 | }; 23 | 24 | } 25 | -------------------------------------------------------------------------------- /src/petuum_ps/thread/bg_oplog.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #include 8 | #include 9 | 10 | namespace petuum { 11 | 12 | class BgOpLog : boost::noncopyable { 13 | public: 14 | BgOpLog(){} 15 | 16 | ~BgOpLog() { 17 | for (auto iter = table_oplog_map_.begin(); iter != table_oplog_map_.end(); 18 | iter++) { 19 | delete iter->second; 20 | } 21 | } 22 | 23 | // takes ownership of the oplog partition 24 | void Add(int32_t table_id, BgOpLogPartition *bg_oplog_partition_ptr){ 25 | table_oplog_map_[table_id] = bg_oplog_partition_ptr; 26 | } 27 | 28 | BgOpLogPartition* Get(int32_t table_id) const { 29 | return table_oplog_map_.at(table_id); 30 | } 31 | private: 32 | std::map table_oplog_map_; 33 | 34 | }; 35 | 36 | } 37 | -------------------------------------------------------------------------------- /src/petuum_ps/thread/bg_oplog_partition.hpp: -------------------------------------------------------------------------------- 1 | // Author: jinliang 2 | 3 | #pragma once 4 | 5 | #include 6 | #include 7 | #include 8 | 9 | #include 10 | #include 11 | 12 | namespace petuum { 13 | 14 | class BgOpLogPartition : boost::noncopyable { 15 | public: 16 | BgOpLogPartition(int32_t table_id, size_t update_size, 17 | int32_t my_comm_channel_idx); 18 | ~BgOpLogPartition(); 19 | 20 | AbstractRowOpLog *FindOpLog(int32_t row_id); 21 | void InsertOpLog(int32_t row_id, AbstractRowOpLog *row_oplog); 22 | void SerializeByServer(std::map *bytes_by_server, 23 | bool dense_serialize = false); 24 | private: 25 | std::unordered_map oplog_map_; 26 | const int32_t table_id_; 27 | const size_t update_size_; 28 | const int32_t comm_channel_idx_; 29 | }; 30 | 31 | } // namespace petuum 32 | -------------------------------------------------------------------------------- /src/petuum_ps/thread/bg_worker_group.hpp: -------------------------------------------------------------------------------- 1 | // author: jinliang 2 | #pragma once 3 | 4 | #include 5 | #include 6 | 7 | namespace petuum { 8 | 9 | class BgWorkerGroup { 10 | public: 11 | BgWorkerGroup(std::map *tables); 12 | virtual ~BgWorkerGroup(); 13 | 14 | void Start(); 15 | 16 | void ShutDown(); 17 | void AppThreadRegister(); 18 | void AppThreadDeregister(); 19 | 20 | // Assuming table does not yet exist 21 | bool CreateTable(int32_t table_id, 22 | const ClientTableConfig& table_config); 23 | void WaitCreateTable(); 24 | bool RequestRow(int32_t table_id, int32_t row_id, int32_t clock); 25 | void RequestRowAsync(int32_t table_id, int32_t row_id, int32_t clock, 26 | bool forced); 27 | void GetAsyncRowRequestReply(); 28 | void SignalHandleAppendOnlyBuffer(int32_t table_id, int32_t channel_idx); 29 | 30 | void ClockAllTables(); 31 | void SendOpLogsAllTables(); 32 | 33 | virtual int32_t GetSystemClock(); 34 | virtual void WaitSystemClock(int32_t my_clock); 35 | 36 | protected: 37 | std::map *tables_; 38 | 39 | /* Helper Functions */ 40 | std::vector bg_worker_vec_; 41 | int32_t bg_worker_id_st_; 42 | 43 | pthread_barrier_t init_barrier_; 44 | pthread_barrier_t create_table_barrier_; 45 | 46 | private: 47 | virtual void CreateBgWorkers(); 48 | 49 | }; 50 | 51 | } 52 | -------------------------------------------------------------------------------- /src/petuum_ps/thread/bg_workers.hpp: -------------------------------------------------------------------------------- 1 | // author: jinliang 2 | 3 | #pragma once 4 | 5 | #include 6 | 7 | namespace petuum { 8 | 9 | class BgWorkers { 10 | public: 11 | static void Start(std::map *tables); 12 | 13 | static void ShutDown(); 14 | static void AppThreadRegister(); 15 | static void AppThreadDeregister(); 16 | 17 | // Assuming table does not yet exist 18 | static bool CreateTable(int32_t table_id, 19 | const ClientTableConfig& table_config); 20 | static void WaitCreateTable(); 21 | static bool RequestRow(int32_t table_id, int32_t row_id, int32_t clock); 22 | // If forced is set to true, a row request is forced to send even if 23 | // it exists in the process storage and clock is fresh enough. 24 | static void RequestRowAsync(int32_t table_id, int32_t row_id, int32_t clock, 25 | bool forced); 26 | static void GetAsyncRowRequestReply(); 27 | static void SignalHandleAppendOnlyBuffer(int32_t table_id, int32_t channel_idx); 28 | static void ClockAllTables(); 29 | static void SendOpLogsAllTables(); 30 | 31 | static int32_t GetSystemClock(); 32 | static void WaitSystemClock(int32_t my_clock); 33 | 34 | private: 35 | static BgWorkerGroup *bg_worker_group_; 36 | }; 37 | 38 | } 39 | -------------------------------------------------------------------------------- /src/petuum_ps/thread/oplog_meta.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | namespace petuum { 11 | 12 | class OpLogMeta : boost::noncopyable { 13 | public: 14 | OpLogMeta() { } 15 | 16 | ~OpLogMeta() { } 17 | 18 | TableOpLogMeta *AddTableOpLogMeta(int32_t table_id, 19 | const AbstractRow *sample_row) { 20 | table_oplog_map_.insert( 21 | std::make_pair(table_id, TableOpLogMeta(sample_row))); 22 | 23 | auto oplog_iter = table_oplog_map_.find(table_id); 24 | 25 | return &(oplog_iter->second); 26 | } 27 | 28 | TableOpLogMeta *Get(int32_t table_id) { 29 | auto oplog_iter = table_oplog_map_.find(table_id); 30 | if (oplog_iter == table_oplog_map_.end()) 31 | return 0; 32 | return &(oplog_iter->second); 33 | } 34 | 35 | bool OpLogMetaExists() const { 36 | for (auto oplog_iter = table_oplog_map_.begin(); 37 | oplog_iter != table_oplog_map_.end(); ++oplog_iter) { 38 | if (oplog_iter->second.GetNumRowOpLogs() > 0) 39 | return true; 40 | } 41 | return false; 42 | } 43 | 44 | private: 45 | std::map table_oplog_map_; 46 | }; 47 | 48 | } 49 | -------------------------------------------------------------------------------- /src/petuum_ps/thread/row_oplog_serializer.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sailing-pmls/bosen/06cb58902d011fbea5f9428f10ce30e621492204/src/petuum_ps/thread/row_oplog_serializer.cpp -------------------------------------------------------------------------------- /src/petuum_ps/thread/server_version_mgr.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | namespace petuum { 8 | // Assume that (#versions per iteration) * staleness < 2^32 -1. 9 | class ServerVersionMgr : boost::noncopyable { 10 | public: 11 | ServerVersionMgr(const std::vector &server_ids); 12 | ~ServerVersionMgr() { } 13 | 14 | // Server versions have an upper bound which is the version number 15 | // of the bg worker. Increment that whenever bg worker's version number 16 | // is incremented. 17 | void IncVersionUpperBound(); 18 | 19 | // Set the version number for server - server_id. 20 | // Return the minimum version number if it has changed. 21 | // Server version monotonically increases. 22 | bool SetServerVersion(int32_t server_id, uint32_t version); 23 | uint32_t GetMinVersion(); 24 | uint32_t GetVersionUpperBound(); 25 | private: 26 | bool IsUniqueMin(int32_t server_id); 27 | boost::unordered_map version_map_; 28 | uint32_t version_upper_bound_; 29 | uint32_t min_version_; 30 | }; 31 | } 32 | -------------------------------------------------------------------------------- /src/petuum_ps/thread/ssp_push_bg_worker_group.hpp: -------------------------------------------------------------------------------- 1 | // author: jinliang 2 | 3 | #pragma once 4 | #include 5 | 6 | namespace petuum { 7 | class SSPPushBgWorkerGroup : public BgWorkerGroup { 8 | public: 9 | SSPPushBgWorkerGroup(std::map *tables); 10 | ~SSPPushBgWorkerGroup() { } 11 | 12 | virtual int32_t GetSystemClock(); 13 | virtual void WaitSystemClock(int32_t my_clock); 14 | 15 | private: 16 | virtual void CreateBgWorkers(); 17 | 18 | // Servers might not update all the rows in a client's process cache 19 | // (there could be a row that nobody writes to for many many clocks). 20 | // Therefore, we need a locally shared clock to denote the minimum clock that 21 | // the entire system has reached, which determines if a local thread may read 22 | // a local cached copy of the row. 23 | // Each bg worker maintains a vector clock for servers. The clock denotes the 24 | // clock that the server has reached, that is the minimum clock that the 25 | // server has seen from all clients. 26 | // This bg_server_clock_ contains one clock for each bg worker which is the 27 | // min server clock that the bg worker has seen. 28 | std::mutex system_clock_mtx_; 29 | std::condition_variable system_clock_cv_; 30 | 31 | std::atomic_int_fast32_t system_clock_{0}; 32 | VectorClockMT bg_server_clock_; 33 | 34 | }; 35 | } 36 | -------------------------------------------------------------------------------- /src/petuum_ps/thread/trans_time_estimate.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "petuum_ps/thread/context.hpp" 4 | #include "petuum_ps_common/include/constants.hpp" 5 | 6 | namespace petuum { 7 | 8 | class TransTimeEstimate { 9 | public: 10 | static double EstimateTransMillisec(size_t accum_sent_bytes) { 11 | return (accum_sent_bytes * kNumBitsPerByte) 12 | / GlobalContext::get_bandwidth_mbps() / kOneThousand; 13 | } 14 | }; 15 | 16 | } 17 | -------------------------------------------------------------------------------- /src/petuum_ps_common/client/abstract_table_group.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * abstract_table_group.hpp 3 | * author: jinliang 4 | */ 5 | 6 | #pragma once 7 | 8 | #include 9 | #include 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include 17 | 18 | #include 19 | 20 | namespace petuum { 21 | 22 | class AbstractTableGroup { 23 | public: 24 | AbstractTableGroup() { } 25 | 26 | virtual ~AbstractTableGroup() { } 27 | 28 | virtual bool CreateTable(int32_t table_id, 29 | const ClientTableConfig& table_config) = 0; 30 | 31 | virtual void CreateTableDone() = 0; 32 | 33 | virtual void WaitThreadRegister() = 0; 34 | 35 | virtual AbstractClientTable *GetTableOrDie(int32_t table_id) = 0; 36 | 37 | virtual int32_t RegisterThread() = 0; 38 | 39 | virtual void DeregisterThread() = 0; 40 | 41 | virtual void Clock() = 0; 42 | 43 | virtual void GlobalBarrier() = 0; 44 | }; 45 | 46 | } // namespace petuum 47 | -------------------------------------------------------------------------------- /src/petuum_ps_common/include/constants.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | namespace petuum { 4 | 5 | const size_t kNumBitsPerByte = 8; 6 | const size_t k1_Mi = 1024*1024; 7 | const size_t k1_Ki = 1024; 8 | const float kCuckooExpansionFactor = 1.428; 9 | 10 | const size_t kOneThousand = 1000; 11 | 12 | } 13 | -------------------------------------------------------------------------------- /src/petuum_ps_common/include/host_info.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | namespace petuum { 7 | 8 | struct HostInfo { 9 | int32_t id; 10 | std::string ip; 11 | std::string port; 12 | 13 | HostInfo() {} 14 | ~HostInfo() { } 15 | HostInfo(int32_t _id, std::string _ip, std::string _port): 16 | id(_id), 17 | ip(_ip), 18 | port(_port) {}; 19 | 20 | HostInfo(const HostInfo &other): 21 | id(other.id), 22 | ip(other.ip), 23 | port(other.port) {} 24 | 25 | HostInfo & operator = (const HostInfo &other) { 26 | id = other.id; 27 | ip = other.ip; 28 | port = other.port; 29 | return *this; 30 | } 31 | }; 32 | 33 | } 34 | -------------------------------------------------------------------------------- /src/petuum_ps_common/include/petuum_ps.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | -------------------------------------------------------------------------------- /src/petuum_ps_common/include/ps_table_group.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | namespace petuum { 4 | AbstractTableGroup *PSTableGroup::abstract_table_group_; 5 | } 6 | -------------------------------------------------------------------------------- /src/petuum_ps_common/oplog/abstract_append_only_buffer.hpp: -------------------------------------------------------------------------------- 1 | // Author: Jinliang 2 | #pragma once 3 | 4 | #include 5 | #include 6 | 7 | #include 8 | 9 | namespace petuum { 10 | 11 | class AbstractAppendOnlyBuffer : boost::noncopyable { 12 | public: 13 | AbstractAppendOnlyBuffer(int32_t thread_id, 14 | size_t capacity, size_t update_size): 15 | thread_id_(thread_id), 16 | capacity_(capacity), 17 | size_(0), 18 | update_size_(update_size), 19 | buff_(new uint8_t[capacity]) { } 20 | 21 | virtual ~AbstractAppendOnlyBuffer() { } 22 | 23 | int32_t get_thread_id() const { 24 | return thread_id_; 25 | } 26 | 27 | void ResetSize() { 28 | size_ = 0; 29 | } 30 | 31 | // return true if succeed, otherwise false 32 | virtual bool Inc(int32_t row_id, int32_t col_id, const void *delta) = 0; 33 | virtual bool BatchInc(int32_t row_id, const int32_t *col_ids, 34 | const void *deltas, int32_t num_updates) = 0; 35 | virtual bool DenseBatchInc(int32_t row_id, const void *updates, int32_t index_st, 36 | int32_t num_updates) = 0; 37 | virtual void InitRead() = 0; 38 | virtual const void *Next(int32_t *row_id, int32_t const **col_ids, 39 | int32_t *num_updates) = 0; 40 | 41 | protected: 42 | const int32_t thread_id_; 43 | 44 | const size_t capacity_; 45 | size_t size_; 46 | 47 | const size_t update_size_; 48 | std::unique_ptr buff_; 49 | 50 | uint8_t* read_ptr_; 51 | }; 52 | 53 | } 54 | -------------------------------------------------------------------------------- /src/petuum_ps_common/oplog/batch_inc_append_only_buffer.hpp: -------------------------------------------------------------------------------- 1 | // Author: Jinliang 2 | #pragma once 3 | 4 | #include 5 | #include 6 | 7 | namespace petuum { 8 | 9 | /* 10 | * This is the append-only buffer type that's specially optimized for BATCH_INC 11 | * operations. The buffer memory is organized as a list of 12 | * (row_id, 13 | * number_updates, *int32_t* 14 | * col_ids, 15 | * updates) tuples. 16 | */ 17 | 18 | class BatchIncAppendOnlyBuffer : public AbstractAppendOnlyBuffer { 19 | public: 20 | BatchIncAppendOnlyBuffer(int32_t thread_id, size_t capacity, size_t update_size): 21 | AbstractAppendOnlyBuffer(thread_id, capacity, update_size) { } 22 | 23 | virtual ~BatchIncAppendOnlyBuffer() { } 24 | 25 | // return true if succeed, otherwise false 26 | bool Inc(int32_t row_id, int32_t col_id, const void *delta); 27 | bool BatchInc(int32_t row_id, const int32_t *col_ids, 28 | const void *deltas, int32_t num_updates) ; 29 | bool DenseBatchInc(int32_t row_id, const void *updates, int32_t index_st, 30 | int32_t num_updates); 31 | void InitRead(); 32 | const void *Next(int32_t *row_id, int32_t const **col_ids, 33 | int32_t *num_updates); 34 | }; 35 | 36 | } 37 | -------------------------------------------------------------------------------- /src/petuum_ps_common/oplog/dense_append_only_buffer.hpp: -------------------------------------------------------------------------------- 1 | // Author: Jinliang 2 | #pragma once 3 | 4 | #include 5 | #include 6 | 7 | namespace petuum { 8 | 9 | /* 10 | * This is the append-only buffer type that's specially optimized for BATCH_INC 11 | * operations. The buffer memory is organized as a list of 12 | * (row_id, 13 | * number_updates, *int32_t* 14 | * col_ids, 15 | * updates) tuples. 16 | */ 17 | 18 | class DenseAppendOnlyBuffer : public AbstractAppendOnlyBuffer { 19 | public: 20 | DenseAppendOnlyBuffer(int32_t thread_id, size_t capacity, size_t update_size, 21 | size_t row_capacity): 22 | AbstractAppendOnlyBuffer(thread_id, capacity, update_size), 23 | row_capacity_(row_capacity) { } 24 | 25 | virtual ~DenseAppendOnlyBuffer() { } 26 | 27 | // return true if succeed, otherwise false 28 | bool Inc(int32_t row_id, int32_t col_id, const void *delta); 29 | bool BatchInc(int32_t row_id, const int32_t *col_ids, 30 | const void *deltas, int32_t num_updates) ; 31 | bool DenseBatchInc(int32_t row_id, const void *updates, int32_t index_st, 32 | int32_t num_updates); 33 | void InitRead(); 34 | const void *Next(int32_t *row_id, int32_t const **col_ids, 35 | int32_t *num_updates); 36 | 37 | protected: 38 | size_t row_capacity_; 39 | }; 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/petuum_ps_common/oplog/inc_append_only_buffer.hpp: -------------------------------------------------------------------------------- 1 | // Author: Jinliang 2 | #pragma once 3 | 4 | #include 5 | #include 6 | 7 | namespace petuum { 8 | 9 | /* 10 | * This is the append-only buffer type that's specially optimized for INC 11 | * operations. The buffer memory is organized as a list of (row_id, col_id, 12 | * update) tuples. 13 | */ 14 | 15 | class IncAppendOnlyBuffer : public AbstractAppendOnlyBuffer { 16 | public: 17 | IncAppendOnlyBuffer(int32_t thread_id, size_t capacity, size_t update_size): 18 | AbstractAppendOnlyBuffer(thread_id, capacity, update_size) { } 19 | 20 | virtual ~IncAppendOnlyBuffer() { } 21 | 22 | // return true if succeed, otherwise false 23 | bool Inc(int32_t row_id, int32_t col_id, const void *delta); 24 | bool BatchInc(int32_t row_id, const int32_t *col_ids, 25 | const void *deltas, int32_t num_updates) ; 26 | bool DenseBatchInc(int32_t row_id, const void *updates, int32_t index_st, 27 | int32_t num_updates); 28 | void InitRead(); 29 | const void *Next(int32_t *row_id, int32_t const **col_ids, 30 | int32_t *num_updates); 31 | }; 32 | 33 | } 34 | -------------------------------------------------------------------------------- /src/petuum_ps_common/storage/abstract_process_storage.hpp: -------------------------------------------------------------------------------- 1 | // Author: Jinliang, Yihua Fang 2 | 3 | #pragma once 4 | 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | namespace petuum { 11 | 12 | class AbstractProcessStorage : boost::noncopyable { 13 | public: 14 | // capacity is the upper bound of the number of rows this ProcessStorage 15 | // can store. 16 | AbstractProcessStorage() { } 17 | 18 | virtual ~AbstractProcessStorage() { } 19 | 20 | // Look up for row_id. If found, return a pointer pointing to that row, 21 | // otherwise return 0. 22 | // If the storage is a evictable type, then the row_accessor is set 23 | // to the corresponding row, which maintains the reference count. 24 | // Otherwise, row_accessor is not used and can actually be a NULL pointer. 25 | virtual ClientRow *Find(int32_t row_id, RowAccessor* row_accessor) = 0; 26 | 27 | virtual bool Find(int32_t row_id) = 0; 28 | 29 | // Insert a row, and take ownership of client_row. 30 | // Insertion failes if the row has already existed and return false; otherwise 31 | // return true. 32 | virtual bool Insert(int32_t row_id, ClientRow* client_row) = 0; 33 | }; 34 | 35 | 36 | } // namespace petuum 37 | -------------------------------------------------------------------------------- /src/petuum_ps_common/storage/bounded_dense_process_storage.hpp: -------------------------------------------------------------------------------- 1 | // Author: Jinliang, Yihua Fang 2 | 3 | #pragma once 4 | 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | namespace petuum { 11 | 12 | // BoundedDenseProcessStorage assumes the rows are in a contiguous range 13 | // and the maximum number of rows existed cannot exceeed capacity. 14 | 15 | class BoundedDenseProcessStorage : public AbstractProcessStorage { 16 | public: 17 | typedef std::function CreateClientRowFunc; 18 | 19 | BoundedDenseProcessStorage(size_t capacity, 20 | CreateClientRowFunc CreateClientRow, 21 | int32_t offset); 22 | 23 | ~BoundedDenseProcessStorage(); 24 | 25 | ClientRow *Find(int32_t row_id, RowAccessor* row_accessor); 26 | 27 | bool Find(int32_t row_id); 28 | 29 | // Insert a row, and take ownership of client_row. 30 | // Insertion failes if the row has already existed and return false; otherwise 31 | // return true. 32 | bool Insert(int32_t row_id, ClientRow* client_row); 33 | 34 | private: 35 | inline ClientRow *FindOffset(int32_t row_id); 36 | inline ClientRow *FindNoOffset(int32_t row_id); 37 | 38 | typedef ClientRow* (BoundedDenseProcessStorage::*FindFunc)(int32_t row_id); 39 | 40 | std::vector storage_vec_; 41 | 42 | const size_t capacity_; 43 | CreateClientRowFunc CreateClientRow_; 44 | FindFunc Find_; 45 | const int32_t offset_; 46 | }; 47 | 48 | } // namespace petuum 49 | -------------------------------------------------------------------------------- /src/petuum_ps_common/storage/entry.hpp: -------------------------------------------------------------------------------- 1 | // Author: Dai Wei (wdai@cs.cmu.edu) 2 | // Date: 2014.10.16 3 | 4 | #pragma once 5 | 6 | #include 7 | #include 8 | 9 | namespace petuum { 10 | 11 | // An Entry is an ID (int32_t) value (V) pair. 12 | // 13 | // Comment(wdai): We cannot use std::pair since we use memcpy and memmove and 14 | // std::pair isn't POD. 15 | template 16 | struct Entry { 17 | static_assert(std::is_pod::value, "V must be POD"); 18 | int32_t first; 19 | V second; 20 | }; 21 | 22 | } // namespace petuum 23 | -------------------------------------------------------------------------------- /src/petuum_ps_common/util/class_register.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /* 4 | * 5 | * author: jinliang 6 | */ 7 | 8 | 9 | #include 10 | #include 11 | 12 | namespace petuum { 13 | template 14 | class ClassRegistry { 15 | public: 16 | typedef BaseClass* (*CreateFunc)(); 17 | ClassRegistry(): 18 | default_creator_(0){}; 19 | ~ClassRegistry(){}; 20 | 21 | void SetDefaultCreator(CreateFunc creator){ 22 | default_creator_ = creator; 23 | } 24 | 25 | void AddCreator(int32_t key, CreateFunc creator){ 26 | creator_map_[key] = creator; 27 | } 28 | 29 | BaseClass *CreateObject(int32_t key){ 30 | typename std::map::const_iterator it = 31 | creator_map_.find(key); 32 | if(it == creator_map_.end()){ 33 | if(default_creator_ != 0){ 34 | return (*default_creator_)(); 35 | } 36 | return 0; 37 | }else{ 38 | CreateFunc creator = it->second; 39 | return (*creator)(); 40 | } 41 | } 42 | 43 | static ClassRegistry &GetRegistry() { 44 | static ClassRegistry registry; 45 | return registry; 46 | } 47 | private: 48 | 49 | CreateFunc default_creator_; 50 | std::map creator_map_; 51 | }; 52 | 53 | template 54 | BaseClass *CreateObj(){ 55 | return dynamic_cast(new ImplClass); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/petuum_ps_common/util/high_resolution_timer.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | namespace petuum { 6 | 7 | HighResolutionTimer::HighResolutionTimer() : total_time_(0) { 8 | restart(); 9 | } 10 | 11 | void HighResolutionTimer::restart() { 12 | int status = clock_gettime(CLOCK_MONOTONIC, &start_time_); 13 | CHECK_NE(-1, status) << "Couldn't initialize start_time_"; 14 | } 15 | 16 | double HighResolutionTimer::elapsed() const { 17 | struct timespec now; 18 | int status = clock_gettime(CLOCK_MONOTONIC, &now); 19 | CHECK_NE(-1, status) << "Couldn't get current time."; 20 | 21 | double ret_sec = double(now.tv_sec - start_time_.tv_sec); 22 | double ret_nsec = double(now.tv_nsec - start_time_.tv_nsec); 23 | 24 | while (ret_nsec < 0) { 25 | ret_sec -= 1.0; 26 | ret_nsec += 1e9; 27 | } 28 | double ret = ret_sec + ret_nsec / 1e9; 29 | return total_time_ + ret; 30 | } 31 | 32 | double HighResolutionTimer::elapsed_max() const { 33 | return double((std::numeric_limits::max)()); 34 | } 35 | 36 | double HighResolutionTimer::elapsed_min() const { 37 | return 0.0; 38 | } 39 | 40 | } // namespace petuum 41 | -------------------------------------------------------------------------------- /src/petuum_ps_common/util/high_resolution_timer.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | namespace petuum { 6 | 7 | // This is a simpler implementation of timer to replace 8 | // boost::high_resolution_timer. Code based on 9 | // http://boost.2283326.n4.nabble.com/boost-shared-mutex-performance-td2659061.html 10 | class HighResolutionTimer { 11 | public: 12 | HighResolutionTimer(); 13 | 14 | void restart(); 15 | 16 | // return elapsed time (including previous restart-->pause time) in seconds. 17 | double elapsed() const; 18 | 19 | // return estimated maximum value for elapsed() 20 | double elapsed_max() const; 21 | 22 | // return minimum value for elapsed() 23 | double elapsed_min() const; 24 | 25 | private: 26 | double total_time_; 27 | struct timespec start_time_; 28 | }; 29 | 30 | } // namespace petuum 31 | -------------------------------------------------------------------------------- /src/petuum_ps_common/util/lockable.hpp: -------------------------------------------------------------------------------- 1 | // Author: Dai Wei (wdai@cs.cmu.edu) 2 | // Date: 2014.02.14 3 | 4 | #pragma once 5 | 6 | 7 | namespace petuum { 8 | 9 | // The Lockable concept (implemented as interface/abstract class) describes 10 | // the characteristics of types that provide exclusive blocking semantics for 11 | // execution agents (i.e. threads). 12 | class Lockable { 13 | public: 14 | // Blocks until a lock can be obtained for the current execution agent. If 15 | // an exception is thrown, no lock is obtained. 16 | virtual void lock() = 0; 17 | 18 | // Releases the lock held by the execution agent. Throws no exceptions. 19 | // requires: The current execution agent should hold the lock. 20 | virtual void unlock() = 0; 21 | 22 | // Attempts to acquire the lock for the current execution agent without 23 | // blocking. If an exception is thrown, no lock is obtained. Return true if 24 | // the lock was acquired, false otherwise 25 | virtual bool try_lock() = 0; 26 | 27 | }; 28 | 29 | } // namespace petuum 30 | -------------------------------------------------------------------------------- /src/petuum_ps_common/util/mpmc_queue.hpp: -------------------------------------------------------------------------------- 1 | // Author: Aurick Qiao (aqiao@cs.cmu,edu), Jinliang 2 | // Date: 2014.10.06 3 | 4 | #pragma once 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include 12 | #include 13 | 14 | namespace petuum { 15 | 16 | // MPMCQueue is a multi-producer-multi-consumer bounded buffer. 17 | 18 | template 19 | class MPMCQueue : boost::noncopyable { 20 | public: 21 | MPMCQueue(size_t capacity) 22 | : capacity_(capacity), 23 | size_(0), 24 | buffer_(new T[capacity]) { 25 | begin_ = buffer_.get(); 26 | end_ = begin_; 27 | } 28 | 29 | ~MPMCQueue() { } 30 | 31 | size_t get_size() const { 32 | return size_; 33 | } 34 | 35 | void Push(const T& value) { 36 | std::unique_lock lock(mtx_); 37 | while (size_ == capacity_) cv_.wait(lock); 38 | memcpy(end_, &value, sizeof(T)); 39 | end_++; 40 | size_++; 41 | if (end_ == buffer_.get() + capacity_) end_ = buffer_.get(); 42 | } 43 | 44 | bool Pop(T* value) { 45 | std::unique_lock lock(mtx_); 46 | if (size_ == 0) return false; 47 | if (size_ == capacity_) cv_.notify_all(); 48 | *value = *begin_; 49 | begin_++; 50 | size_--; 51 | if (begin_ == buffer_.get() + capacity_) begin_ = buffer_.get(); 52 | return true; 53 | } 54 | 55 | private: 56 | size_t capacity_, size_; 57 | std::unique_ptr buffer_; 58 | T *begin_, *end_; 59 | std::mutex mtx_; 60 | std::condition_variable cv_; 61 | }; 62 | 63 | } // namespace petuum 64 | -------------------------------------------------------------------------------- /src/petuum_ps_common/util/mt_queue.hpp: -------------------------------------------------------------------------------- 1 | // Author: Dai Wei (wdai@cs.cmu.edu) 2 | // Date: 2014.01.31 3 | 4 | #pragma once 5 | 6 | #include 7 | #include 8 | 9 | namespace petuum { 10 | 11 | // Wrap around std::queue and provide thread safety (MT = multi-threaded). 12 | template 13 | class MTQueue { 14 | public: 15 | MTQueue() { } 16 | 17 | // Return 0 if it's empty, 1 if not. val is unset if returning 0. Note the 18 | // different semantics than std::queue::pop. 19 | int pop(T* val) { 20 | std::lock_guard lock(mutex_); 21 | if (!q_.empty()) { 22 | *val = q_.front(); 23 | q_.pop(); 24 | return 1; 25 | } 26 | return 0; 27 | } 28 | 29 | void push(const T& val) { 30 | std::lock_guard lock(mutex_); 31 | q_.push(val); 32 | } 33 | 34 | private: 35 | std::queue q_; 36 | std::mutex mutex_; 37 | }; 38 | 39 | 40 | } // namespace petuum 41 | -------------------------------------------------------------------------------- /src/petuum_ps_common/util/thread.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | namespace petuum { 6 | class Thread { 7 | public: 8 | Thread() { } 9 | 10 | virtual ~Thread() { } 11 | 12 | virtual void *operator() () { 13 | return 0; 14 | } 15 | 16 | int Start() { 17 | InitWhenStart(); 18 | return pthread_create(&thr_, NULL, InternalStart, this); 19 | } 20 | 21 | void Join() { 22 | pthread_join(thr_, NULL); 23 | } 24 | 25 | protected: 26 | virtual void InitWhenStart() { } 27 | 28 | private: 29 | static void *InternalStart(void *thread) { 30 | Thread *t = reinterpret_cast(thread); 31 | return t->operator()(); 32 | } 33 | 34 | pthread_t thr_; 35 | }; 36 | } 37 | -------------------------------------------------------------------------------- /src/petuum_ps_common/util/thread_group.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | namespace petuum { 6 | 7 | template 8 | class ThreadGroup { 9 | public: 10 | virtual ~ThreadGroup() { 11 | for (auto &thread : threads_) { 12 | delete thread; 13 | } 14 | } 15 | 16 | void StartAll() { 17 | for (auto &thread : threads_) { 18 | thread->Start(); 19 | } 20 | } 21 | 22 | void JoinAll() { 23 | for (auto &thread : threads_) { 24 | thread->Join(); 25 | } 26 | } 27 | 28 | protected: 29 | ThreadGroup(size_t num_threads): 30 | threads_(num_threads) { } 31 | 32 | std::vector threads_; 33 | }; 34 | } 35 | -------------------------------------------------------------------------------- /src/petuum_ps_common/util/timer_thr.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | #include 5 | 6 | namespace petuum { 7 | 8 | NanoTimer::NanoTimer(): 9 | started_(false){} 10 | 11 | int NanoTimer::Start(int32_t _interval, TimerHandler _handler, 12 | void *_handler_argu){ 13 | 14 | if(started_) return -1; 15 | if(_interval <= 0) return -1; 16 | 17 | tinfo_.interval_ = _interval; 18 | tinfo_.callback_ = _handler; 19 | tinfo_.handler_argu_ = _handler_argu; 20 | 21 | int ret = pthread_create(&thr_, NULL, TimerThreadMain, &tinfo_); 22 | if(ret != 0) return -1; 23 | started_ = true; 24 | 25 | return 0; 26 | } 27 | 28 | int NanoTimer::Stop(){ 29 | if(!started_) return -1; 30 | pthread_join(thr_, NULL); 31 | started_ = false; 32 | return 0; 33 | } 34 | 35 | void *NanoTimer::TimerThreadMain(void *argu){ 36 | TimerThrInfo *tinfo = reinterpret_cast(argu); 37 | int32_t interval = tinfo->interval_; 38 | while(1){ 39 | timespec req; 40 | req.tv_sec = 0; 41 | req.tv_nsec = interval; 42 | timespec rem; 43 | 44 | nanosleep(&req, &rem); 45 | 46 | interval = tinfo->callback_(tinfo->handler_argu_, rem.tv_nsec); 47 | if(interval <= 0) break; 48 | } 49 | return 0; 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /src/petuum_ps_common/util/timer_thr.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | namespace petuum { 9 | 10 | class NanoTimer{ 11 | 12 | // true if time continue, false if timer stop 13 | typedef int32_t (*TimerHandler)(void *, int32_t); 14 | 15 | struct TimerThrInfo{ 16 | int32_t interval_; 17 | TimerHandler callback_; 18 | void *handler_argu_; 19 | }; 20 | 21 | public: 22 | NanoTimer(); 23 | 24 | int Start(int32_t _interval, TimerHandler _handler, void *_handler_argu); 25 | 26 | int Stop(); 27 | 28 | private: 29 | static void *TimerThreadMain(void *argu); 30 | 31 | pthread_t thr_; 32 | TimerThrInfo tinfo_; 33 | bool started_; 34 | 35 | }; 36 | 37 | } 38 | -------------------------------------------------------------------------------- /src/petuum_ps_common/util/utils.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | namespace petuum { 10 | 11 | void GetHostInfos(std::string server_file, 12 | std::map *host_map) { 13 | std::map& servers = *host_map; 14 | 15 | std::ifstream input(server_file.c_str()); 16 | std::string line; 17 | while(std::getline(input, line)){ 18 | 19 | size_t pos = line.find_first_of("\t "); 20 | std::string idstr = line.substr(0, pos); 21 | 22 | size_t pos_ip = line.find_first_of("\t ", pos + 1); 23 | std::string ip = line.substr(pos + 1, pos_ip - pos - 1); 24 | std::string port = line.substr(pos_ip + 1); 25 | 26 | int32_t id = atoi(idstr.c_str()); 27 | servers.insert(std::make_pair(id, petuum::HostInfo(id, ip, port))); 28 | //VLOG(0) << "get server: " << id << ":" << ip << ":" << port; 29 | } 30 | input.close(); 31 | } 32 | 33 | // assuming the namenode id is 0 34 | void GetServerIDsFromHostMap(std::vector *server_ids, 35 | const std::map& host_map){ 36 | 37 | int32_t num_servers = host_map.size() - 1; 38 | server_ids->resize(num_servers); 39 | int32_t i = 0; 40 | 41 | for (auto host_info_iter = host_map.cbegin(); 42 | host_info_iter != host_map.cend(); host_info_iter++) { 43 | if (host_info_iter->first == 0) 44 | continue; 45 | (*server_ids)[i] = host_info_iter->first; 46 | ++i; 47 | } 48 | } 49 | 50 | 51 | } // namespace petuum 52 | -------------------------------------------------------------------------------- /src/petuum_ps_common/util/utils.hpp: -------------------------------------------------------------------------------- 1 | 2 | #pragma once 3 | 4 | #include 5 | #include 6 | 7 | #include 8 | 9 | namespace petuum { 10 | // Read in a file containing list of servers. 'server_file' need to have the 11 | // following line structure: 12 | // 13 | // (tab in as deliminator) 1 128.0.1.1 80 14 | // 15 | // Note that the first line of the file will be considered as name node. 16 | void GetHostInfos(std::string server_file, 17 | std::map *host_map); 18 | 19 | void GetServerIDsFromHostMap(std::vector *server_ids, 20 | const std::map & host_map); 21 | 22 | } // namespace petuum 23 | -------------------------------------------------------------------------------- /src/petuum_ps_common/util/vector_clock.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | namespace petuum { 8 | 9 | // VectorClock manages a vector of clocks and maintains the minimum of these 10 | // clocks. This class is single thread (ST) only. 11 | class VectorClock { 12 | public: 13 | VectorClock(); 14 | // Initialize client_ids.size() client clocks with all of them at time 0. 15 | explicit VectorClock(const std::vector& ids); 16 | 17 | // Add a clock in vector clock with initial timestampe. id must be unique. 18 | // Return 0 on success, negatives on error (e.g., duplicated id). 19 | virtual void AddClock(int32_t id, int32_t clock = 0); 20 | 21 | // Increment client's clock. Accordingly update slowest_client_clock_. 22 | // Return the minimum clock if client_id is the slowest thread; 0 if not; 23 | // negatives for error; 24 | virtual int32_t Tick(int32_t id); 25 | virtual int32_t TickUntil(int32_t id, int32_t clock); 26 | 27 | // Getters 28 | virtual int32_t get_clock(int32_t id) const; 29 | virtual int32_t get_min_clock() const; 30 | 31 | private: 32 | // If the tick of this client will change the slowest_client_clock_, then 33 | // it returns true. 34 | bool IsUniqueMin(int32_t id); 35 | 36 | boost::unordered_map vec_clock_; 37 | 38 | // Slowest client clock 39 | int32_t min_clock_; 40 | }; 41 | 42 | } // namespace petuum 43 | -------------------------------------------------------------------------------- /src/petuum_ps_common/util/vector_clock_mt.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | namespace petuum { 5 | 6 | VectorClockMT::VectorClockMT(): 7 | VectorClock(){} 8 | 9 | VectorClockMT::VectorClockMT(const std::vector& ids) : 10 | VectorClock(ids) { } 11 | 12 | void VectorClockMT::AddClock(int32_t id, int32_t clock) { 13 | std::unique_lock write_lock(mutex_); 14 | VectorClock::AddClock(id, clock); 15 | } 16 | 17 | int32_t VectorClockMT::Tick(int32_t id) { 18 | std::unique_lock write_lock(mutex_); 19 | return VectorClock::Tick(id); 20 | } 21 | 22 | int32_t VectorClockMT::TickUntil(int32_t id, int32_t clock) { 23 | std::unique_lock write_lock(mutex_); 24 | return VectorClock::TickUntil(id, clock); 25 | } 26 | 27 | int32_t VectorClockMT::get_clock(int32_t id) const { 28 | std::unique_lock read_lock(mutex_); 29 | return VectorClock::get_clock(id); 30 | } 31 | 32 | int32_t VectorClockMT::get_min_clock() const { 33 | std::unique_lock read_lock(mutex_); 34 | return VectorClock::get_min_clock(); 35 | } 36 | 37 | } // namespace petuum 38 | -------------------------------------------------------------------------------- /src/petuum_ps_common/util/vector_clock_mt.hpp: -------------------------------------------------------------------------------- 1 | // author: jinliang 2 | 3 | #pragma once 4 | 5 | #include 6 | #include 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | namespace petuum { 15 | 16 | // VectorClock is a thread-safe extension of VectorClockST. 17 | class VectorClockMT : public VectorClock, boost::noncopyable { 18 | public: 19 | VectorClockMT(); 20 | explicit VectorClockMT(const std::vector& ids); 21 | 22 | // Override VectorClock 23 | void AddClock(int32_t id, int32_t clock = 0); 24 | int32_t Tick(int32_t id); 25 | int32_t TickUntil(int32_t id, int32_t clock); 26 | 27 | // Accessor to a particular clock. 28 | int32_t get_clock(int32_t id) const; 29 | int32_t get_min_clock() const; 30 | 31 | private: 32 | // Lock for slowest record 33 | mutable SharedMutex mutex_; 34 | }; 35 | 36 | } // namespace petuum 37 | -------------------------------------------------------------------------------- /src/petuum_ps_common/util/vector_map.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sailing-pmls/bosen/06cb58902d011fbea5f9428f10ce30e621492204/src/petuum_ps_common/util/vector_map.hpp -------------------------------------------------------------------------------- /src/yarn/README.md: -------------------------------------------------------------------------------- 1 | This is a gradle project, you must install gradle to constuct the project and export jar package. 2 | To install gradle, you can refer to https://gradle.org 3 | 4 | run 5 | ``` 6 | gradle build 7 | ``` 8 | to export a jar package (in ./build/libs) for running. 9 | -------------------------------------------------------------------------------- /src/yarn/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'java' 2 | 3 | version = '0.5' 4 | 5 | repositories { 6 | jcenter() 7 | mavenCentral() 8 | } 9 | 10 | dependencies { 11 | compile 'org.slf4j:jcl-over-slf4j:1.7.2' 12 | compile 'commons-io:commons-io:2.1' 13 | compile 'org.apache.hadoop:hadoop-yarn-server-tests:2.4.0' 14 | compile 'org.apache.hadoop:hadoop-common:2.4.0' 15 | compile 'org.apache.hadoop:hadoop-mapreduce-client-core:2.4.0' 16 | compile 'org.apache.hadoop:hadoop-yarn-client:2.4.0' 17 | compile 'junit:junit:4.10' 18 | } 19 | -------------------------------------------------------------------------------- /src/yarn/gradle.properties.template: -------------------------------------------------------------------------------- 1 | systemProp.https.proxyHost=ops 2 | systemProp.https.proxyPort=8888 3 | -------------------------------------------------------------------------------- /src/yarn/src/test/java/cn/edu/buaa/act/petuumOnYarn/TestPetuumOnYarn.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package cn.edu.buaa.act.petuumOnYarn; 20 | 21 | public class TestPetuumOnYarn { 22 | } 23 | 24 | -------------------------------------------------------------------------------- /src/yarn/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | # log4j configuration used during build and unit tests 14 | 15 | log4j.rootLogger=info,stdout 16 | log4j.threshhold=ALL 17 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 18 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 19 | log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n 20 | --------------------------------------------------------------------------------