├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── actor.opam ├── bin ├── actor_manager.ml ├── actor_worker.ml └── jbuild ├── data ├── kmeans.data └── wordcount.data ├── jbuild-ignore ├── jbuild-workspace ├── lib ├── actor.ml ├── actor_barrier.ml ├── actor_config.ml ├── actor_dag.ml ├── actor_mapre.ml ├── actor_mapre.mli ├── actor_mapreclient.ml ├── actor_mapreserver.ml ├── actor_memory.ml ├── actor_param.ml ├── actor_param.mli ├── actor_param_sgd.ml ├── actor_paramclient.ml ├── actor_paramserver.ml ├── actor_peer.ml ├── actor_peer.mli ├── actor_peer_lda.ml ├── actor_peer_sgd0.ml ├── actor_peer_sgd1.ml ├── actor_peerclient.ml ├── actor_peerserver.ml ├── actor_service.ml ├── actor_storage.ml ├── actor_types.ml ├── actor_utils.ml └── jbuild └── test ├── jbuild ├── test_context.ml ├── test_coordinate.ml ├── test_irmin.ml ├── test_kmeans.ml ├── test_owl_parallel.ml ├── test_param_sgd.ml ├── test_parameter.ml ├── test_peer.ml ├── test_peer_lda.ml ├── test_peer_sgd.ml ├── test_serialise.ml └── test_wordcount.ml /.gitignore: -------------------------------------------------------------------------------- 1 | *.annot 2 | *.cmo 3 | *.cma 4 | *.cmi 5 | *.a 6 | *.o 7 | *.cmx 8 | *.cmxs 9 | *.cmxa 10 | 11 | # ocamlbuild working directory 12 | _build/ 13 | 14 | # ocamlbuild targets 15 | *.byte 16 | *.native 17 | 18 | # oasis generated files 19 | setup.data 20 | setup.log 21 | 22 | # jbuilder generated files 23 | .merlin 24 | *.install -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Liang 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: all 2 | all: build 3 | 4 | .PHONY: depend depends 5 | depend depends: 6 | jbuilder external-lib-deps --missing @install @runtest 7 | 8 | .PHONY: build 9 | build: depends 10 | jbuilder build @install 11 | 12 | .PHONY: test 13 | test: depends 14 | jbuilder runtest -j 1 --no-buffer -p owl 15 | 16 | .PHONY: clean 17 | clean: 18 | jbuilder clean 19 | 20 | .PHONY: install 21 | install: build 22 | dune install 23 | 24 | .PHONY: uninstall 25 | uninstall: 26 | dune uninstall 27 | 28 | .PHONY: doc 29 | doc: 30 | dune build @doc 31 | 32 | .PHONY: cleanall 33 | cleanall: 34 | dune uninstall && dune clean 35 | $(RM) -r $(find . -name .merlin) 36 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # OCaml Distributed Data Processing 2 | 3 | A distributed data processing system developed in OCaml 4 | 5 | # Todo 6 | 7 | How to express arbitrary DAG? How to express loop? Apply function. 8 | 9 | Interface to Irmin or HDFS to provide persistent storage 10 | 11 | Test delay-bounded and error-bounded barrier 12 | 13 | Split Context module into server and client two modules 14 | 15 | Implement parameter.mli 16 | 17 | Implement barrier control in parameter modules 18 | 19 | Rename ... DataContext and ModelContext? 20 | 21 | Implement Coordinate Descent in model parallel ... 22 | 23 | Enhance Mapreduce engine, incorporate with owl. 24 | 25 | Add techreport based on the barrier control. 26 | 27 | 28 | # How to compile & run it? 29 | 30 | To compile and build the system, you do not have to install all the software yourself. You can simply pull a ready-made container to set up development environment. 31 | 32 | ```bash 33 | docker pull ryanrhymes/actor 34 | ``` 35 | 36 | Then you can start the container by 37 | 38 | ```bash 39 | docker run -t -i ryanrhymes/actor:latest /bin/bash 40 | ``` 41 | 42 | After the container starts, go to the home director, clone the git repository. 43 | 44 | ```bash 45 | git clone https://github.com/ryanrhymes/actor.git 46 | ``` 47 | 48 | Then you can compile and build the system. 49 | 50 | ```bash 51 | make oasis && make 52 | ``` 53 | -------------------------------------------------------------------------------- /actor.opam: -------------------------------------------------------------------------------- 1 | opam-version: "1.2" 2 | maintainer: "Liang Wang (ryanrhymes@gmail.com)" 3 | authors: [ "Liang Wang (ryanrhymes@gmail.com)" ] 4 | license: "MIT" 5 | homepage: "https://github.com/ryanrhymes/actor/" 6 | dev-repo: "git+https://github.com/ryanrhymes/actor.git" 7 | bug-reports: "https://github.com/ryanrhymes/actor/issues" 8 | doc: "http://www.cl.cam.ac.uk/~lw525/" 9 | 10 | version: "dev" 11 | 12 | build: [ 13 | ["dune" "subst" "-n" name] {pinned} 14 | ["dune" "build" "-p" name "-j" jobs] 15 | ] 16 | build-test: [["dune" "runtest" "-p" name "-j" jobs]] 17 | 18 | depexts: [ 19 | "zmq" 20 | ] 21 | 22 | depends: [ 23 | "dune" {build} 24 | "ocamlgraph" 25 | "owl" 26 | "zmq" 27 | ] 28 | available: [ ocaml-version >= "4.04.0" ] 29 | -------------------------------------------------------------------------------- /bin/actor_manager.ml: -------------------------------------------------------------------------------- 1 | (* 2 | * Actor - Parallel & Distributed Engine of Owl System 3 | * Copyright (c) 2016-2018 Liang Wang 4 | *) 5 | 6 | (* Manager: keeps running to manage a group of actors *) 7 | 8 | open Actor_types 9 | 10 | module Workers = struct 11 | let _workers = ref StrMap.empty 12 | 13 | let create id addr = { 14 | id = id; 15 | addr = addr; 16 | last_seen = Unix.time () 17 | } 18 | 19 | let add id addr = _workers := StrMap.add id (create id addr) !_workers 20 | let remove id = _workers := StrMap.remove id !_workers 21 | let mem id = StrMap.mem id !_workers 22 | let to_list () = StrMap.fold (fun _k v l -> l @ [v]) !_workers [] 23 | let addrs () = StrMap.fold (fun _k v l -> l @ [v.addr]) !_workers [] 24 | end 25 | 26 | let addr = Actor_config.manager_addr 27 | let myid = Actor_config.manager_id 28 | 29 | let process r m = 30 | match m.typ with 31 | | User_Reg -> ( 32 | let uid, addr = m.par.(0), m.par.(1) in 33 | if Workers.mem uid = false then 34 | Owl_log.info "%s" (uid ^ " @ " ^ addr); 35 | Workers.add uid addr; 36 | Actor_utils.send r OK [||]; 37 | ) 38 | | Job_Reg -> ( 39 | let master, jid = m.par.(0), m.par.(1) in 40 | if Actor_service.mem jid = false then ( 41 | Actor_service.add jid master; 42 | (* FIXME: currently send back all nodes as workers *) 43 | let addrs = Marshal.to_string (Workers.addrs ()) [] in 44 | Actor_utils.send r Job_Master [|addrs|] ) 45 | else 46 | let master = (Actor_service.find jid).master in 47 | Actor_utils.send r Job_Worker [|master|] 48 | ) 49 | | Heartbeat -> ( 50 | Owl_log.info "%s" ("heartbeat @ " ^ m.par.(0)); 51 | Workers.add m.par.(0) m.par.(1); 52 | Actor_utils.send r OK [||]; 53 | ) 54 | | P2P_Reg -> ( 55 | let addr, jid = m.par.(0), m.par.(1) in 56 | Owl_log.info "p2p @ %s job:%s" addr jid; 57 | if Actor_service.mem jid = false then Actor_service.add jid ""; 58 | let peers = Actor_service.choose_workers jid 10 in 59 | let peers = Marshal.to_string peers [] in 60 | Actor_service.add_worker jid addr; 61 | Actor_utils.send r OK [|peers|]; 62 | ) 63 | | _ -> ( 64 | Owl_log.error "unknown message type" 65 | ) 66 | 67 | let run _id addr = 68 | let _ztx = ZMQ.Context.create () in 69 | let rep = ZMQ.Socket.create _ztx ZMQ.Socket.rep in 70 | ZMQ.Socket.bind rep addr; 71 | while true do 72 | let m = of_msg (ZMQ.Socket.recv rep) in 73 | process rep m; 74 | done; 75 | ZMQ.Socket.close rep; 76 | ZMQ.Context.terminate _ztx 77 | 78 | let install_app _x = None 79 | 80 | let _ = run myid addr 81 | -------------------------------------------------------------------------------- /bin/actor_worker.ml: -------------------------------------------------------------------------------- 1 | (* 2 | * Actor - Parallel & Distributed Engine of Owl System 3 | * Copyright (c) 2016-2018 Liang Wang 4 | *) 5 | 6 | (* Actor: connect to Manager, represent a working node/actor. *) 7 | 8 | open Actor_types 9 | 10 | let manager = Actor_config.manager_addr 11 | let addr = "tcp://127.0.0.1:" ^ (string_of_int (Random.int 10000 + 50000)) 12 | let myid = "worker_" ^ (string_of_int (Random.int 9000 + 1000)) 13 | let _ztx = ZMQ.Context.create () 14 | 15 | let register req id u_addr m_addr = 16 | Owl_log.info "%s" ("register -> " ^ m_addr); 17 | Actor_utils.send req User_Reg [|id; u_addr|]; 18 | ignore (ZMQ.Socket.recv req) 19 | 20 | let heartbeat req id u_addr m_addr = 21 | Owl_log.info "%s" ("heartbeat -> " ^ m_addr); 22 | Actor_utils.send req Heartbeat [|id; u_addr|]; 23 | ignore (ZMQ.Socket.recv req) 24 | 25 | let start_app app arg = 26 | Owl_log.info "%s" ("starting job " ^ app); 27 | match Unix.fork () with 28 | | 0 -> if Unix.fork () = 0 then Unix.execv app arg else exit 0 29 | | _p -> ignore(Unix.wait ()) 30 | 31 | let deploy_app _x = Owl_log.error "cannot find %s" _x 32 | 33 | let run id u_addr m_addr = 34 | (* set up connection to manager *) 35 | let req = ZMQ.Socket.create _ztx ZMQ.Socket.req in 36 | ZMQ.Socket.connect req m_addr; 37 | register req myid u_addr m_addr; 38 | (* set up local service *) 39 | let rep = ZMQ.Socket.create _ztx ZMQ.Socket.rep in 40 | ZMQ.Socket.bind rep u_addr; 41 | while true do 42 | ZMQ.Socket.set_receive_timeout rep (300 * 1000); 43 | try let m = of_msg (ZMQ.Socket.recv rep) in 44 | match m.typ with 45 | | Job_Create -> ( 46 | let app = m.par.(1) in 47 | let arg = Marshal.from_string m.par.(2) 0 in 48 | Owl_log.info "%s" (app ^ " <- " ^ m.par.(0)); 49 | ZMQ.Socket.send rep (Marshal.to_string OK []); 50 | match Sys.file_exists app with 51 | | true -> start_app app arg 52 | | false -> deploy_app app 53 | ) 54 | | _ -> () 55 | with 56 | | Unix.Unix_error (_,_,_) -> heartbeat req id u_addr m_addr 57 | | ZMQ.ZMQ_exception (_,s) -> Owl_log.error "%s" s 58 | | _exn -> Owl_log.error "unknown error" 59 | done; 60 | ZMQ.Socket.close rep; 61 | ZMQ.Socket.close req; 62 | ZMQ.Context.terminate _ztx 63 | 64 | let () = run myid addr manager 65 | -------------------------------------------------------------------------------- /bin/jbuild: -------------------------------------------------------------------------------- 1 | (jbuild_version 1) 2 | 3 | (executables 4 | ((public_names (actor_manager actor_worker)) 5 | (names (actor_manager actor_worker)) 6 | (libraries ( 7 | actor 8 | )))) 9 | -------------------------------------------------------------------------------- /data/kmeans.data: -------------------------------------------------------------------------------- 1 | 0.44559,0.24279 2 | 0.64631,0.4424 3 | 0.70936,0.6878 4 | 0.75469,0.35923 5 | 0.27603,0.73634 6 | 0.6797,0.39471 7 | 0.6551,0.68342 8 | 0.16261,0.70405 9 | 0.119,0.44231 10 | 0.49836,0.019578 11 | 0.95974,0.33086 12 | 0.34039,0.42431 13 | 0.58527,0.27027 14 | 0.22381,0.19705 15 | 0.75127,0.82172 16 | 0.2551,0.42992 17 | 0.50596,0.88777 18 | 0.69908,0.39118 19 | 0.8909,0.76911 20 | 0.95929,0.39679 21 | 0.54722,0.80851 22 | 0.13862,0.75508 23 | 0.14929,0.3774 24 | 0.25751,0.21602 25 | 0.84072,0.79041 26 | 0.25428,0.9493 27 | 0.81428,0.32757 28 | 0.24352,0.67126 29 | 0.92926,0.43864 30 | 0.34998,0.8335 31 | 0.1966,0.76885 32 | 0.25108,0.16725 33 | 0.61604,0.86198 34 | 0.47329,0.98987 35 | 0.35166,0.51442 36 | 0.83083,0.88428 37 | 0.58526,0.58803 38 | 0.54972,0.15475 39 | 0.91719,0.19986 40 | 0.28584,0.40695 41 | 0.7572,0.74871 42 | 0.75373,0.82558 43 | 0.38045,0.78996 44 | 0.56782,0.31852 45 | 0.075854,0.53406 46 | 0.05395,0.089951 47 | 0.5308,0.11171 48 | 0.77917,0.13629 49 | 0.93401,0.67865 50 | 0.12991,0.49518 51 | 0.56882,0.18971 52 | 0.46939,0.49501 53 | 0.011902,0.14761 54 | 0.33712,0.054974 55 | 0.16218,0.85071 56 | 0.79428,0.56056 57 | 0.31122,0.92961 58 | 0.52853,0.69667 59 | 0.16565,0.58279 60 | 0.60198,0.8154 61 | 0.26297,0.87901 62 | 0.65408,0.98891 63 | 0.68921,0.00052238 64 | 0.74815,0.86544 65 | 0.45054,0.61257 66 | 0.083821,0.98995 67 | 0.22898,0.52768 68 | 0.91334,0.47952 69 | 0.15238,0.80135 70 | 0.82582,0.22784 71 | 0.53834,0.49809 72 | 0.99613,0.90085 73 | 0.078176,0.57466 74 | 0.44268,0.84518 75 | 0.10665,0.73864 76 | 0.9619,0.58599 77 | 0.0046342,0.24673 78 | 0.77491,0.66642 79 | 0.8173,0.083483 80 | 0.86869,0.62596 81 | 0.084436,0.66094 82 | 0.39978,0.72975 83 | 0.25987,0.89075 84 | 0.80007,0.9823 85 | 0.43141,0.76903 86 | 0.91065,0.58145 87 | 0.18185,0.92831 88 | 0.2638,0.58009 89 | 0.14554,0.016983 90 | 0.13607,0.12086 91 | 0.86929,0.86271 92 | 0.5797,0.4843 93 | 0.54986,0.84486 94 | 0.14495,0.20941 95 | 0.85303,0.55229 96 | 0.62206,0.62988 97 | 0.35095,0.031991 98 | 0.51325,0.61471 99 | 0.40181,0.36241 100 | 0.075967,0.049533 101 | 0.23992,0.48957 102 | 0.12332,0.19251 103 | 0.18391,0.12308 104 | 0.23995,0.20549 105 | 0.41727,0.14651 106 | 0.049654,0.18907 107 | 0.90272,0.042652 108 | 0.94479,0.6352 109 | 0.49086,0.28187 110 | 0.48925,0.5386 111 | 0.33772,0.69516 112 | 0.90005,0.49912 113 | 0.36925,0.5358 114 | 0.1112,0.44518 115 | 0.78025,0.12393 116 | 0.38974,0.49036 117 | 0.24169,0.853 118 | 0.40391,0.87393 119 | 0.096455,0.27029 120 | 0.13197,0.20846 121 | 0.94205,0.56498 122 | 0.95613,0.64031 123 | 0.57521,0.41703 124 | 0.05978,0.20598 125 | 0.23478,0.94793 126 | 0.35316,0.082071 127 | 0.82119,0.10571 128 | 0.015403,0.14204 129 | 0.043024,0.16646 130 | 0.16899,0.62096 131 | 0.64912,0.57371 132 | 0.73172,0.052078 133 | 0.64775,0.9312 134 | 0.45092,0.72866 135 | 0.54701,0.73784 136 | 0.29632,0.063405 137 | 0.74469,0.86044 138 | 0.18896,0.93441 139 | 0.68678,0.9844 140 | 0.18351,0.85894 141 | 0.36848,0.78556 142 | 0.62562,0.51338 143 | 0.78023,0.1776 144 | 0.081126,0.39859 145 | 0.92939,0.13393 146 | 0.77571,0.03089 147 | 0.48679,0.93914 148 | 0.43586,0.30131 149 | 0.44678,0.29553 150 | 0.30635,0.33294 151 | 0.50851,0.46707 152 | 0.51077,0.6482 153 | 0.81763,0.025228 154 | 0.79483,0.84221 155 | 0.64432,0.55903 156 | 0.37861,0.8541 157 | 0.81158,0.34788 158 | 0.53283,0.44603 159 | 0.35073,0.054239 160 | 0.939,0.17711 161 | 0.87594,0.66281 162 | 0.55016,0.33083 163 | 0.62248,0.89849 164 | 0.58704,0.11816 165 | 0.20774,0.98842 166 | 0.30125,0.53998 167 | 0.47092,0.70692 168 | 0.23049,0.99949 169 | 0.84431,0.28785 170 | 0.19476,0.41452 171 | 0.22592,0.46484 172 | 0.17071,0.76396 173 | 0.22766,0.8182 174 | 0.4357,0.10022 175 | 0.3111,0.17812 176 | 0.92338,0.35963 177 | 0.43021,0.056705 178 | 0.18482,0.52189 179 | 0.90488,0.33585 180 | 0.97975,0.17567 181 | 0.43887,0.20895 182 | 0.11112,0.90515 183 | 0.25806,0.67539 184 | 0.40872,0.46847 185 | 0.5949,0.91213 186 | 0.26221,0.10401 187 | 0.60284,0.74555 188 | 0.71122,0.73627 189 | 0.22175,0.56186 190 | 0.11742,0.18419 191 | 0.29668,0.59721 192 | 0.31878,0.29994 193 | 0.42417,0.13412 194 | 0.50786,0.2126 195 | 0.085516,0.89494 196 | 0.26248,0.071453 197 | 0.80101,0.24249 198 | 0.02922,0.053754 199 | 0.92885,0.44172 200 | 0.73033,0.013283 201 | 0.48861,0.89719 202 | 0.57853,0.19666 203 | 0.23728,0.093371 204 | 0.45885,0.30737 205 | 0.96309,0.45606 206 | 0.54681,0.10167 207 | 0.52114,0.99539 208 | 0.23159,0.33209 209 | 0.4889,0.29735 210 | 0.62406,0.062045 211 | 0.67914,0.29824 212 | 0.39552,0.046351 213 | 0.36744,0.50543 214 | 0.98798,0.76143 215 | 0.037739,0.63107 216 | 0.88517,0.089892 217 | 0.91329,0.080862 218 | 0.79618,0.77724 219 | 0.098712,0.90513 220 | 0.26187,0.53377 221 | 0.33536,0.10915 222 | 0.67973,0.82581 223 | 0.13655,0.3381 224 | 0.72123,0.29397 225 | 0.10676,0.74631 226 | 0.65376,0.010337 227 | 0.49417,0.048447 228 | 0.77905,0.66792 229 | 0.71504,0.60347 230 | 0.90372,0.5261 231 | 0.89092,0.72971 232 | 0.33416,0.70725 233 | 0.69875,0.78138 234 | 0.19781,0.28798 235 | 0.030541,0.69253 236 | 0.74407,0.55667 237 | 0.50002,0.39652 238 | 0.47992,0.061591 239 | 0.90472,0.78018 240 | 0.60987,0.33758 241 | 0.61767,0.60787 242 | 0.85944,0.74125 243 | 0.80549,0.10481 244 | 0.57672,0.12789 245 | 0.18292,0.54954 246 | 0.23993,0.48523 247 | 0.88651,0.89048 248 | 0.028674,0.79896 249 | 0.4899,0.73434 250 | 0.16793,0.051332 251 | 0.97868,0.072885 252 | 0.71269,0.088527 253 | 0.50047,0.79835 254 | 0.47109,0.94301 255 | 0.059619,0.68372 256 | 0.68197,0.13208 257 | 0.042431,0.72272 258 | 0.071445,0.11035 259 | 0.52165,0.11749 260 | 0.09673,0.64072 261 | 0.81815,0.32881 262 | 0.81755,0.65381 263 | 0.72244,0.74913 264 | 0.14987,0.58319 265 | 0.65961,0.74003 266 | 0.51859,0.23483 267 | 0.97297,0.73496 268 | 0.64899,0.9706 269 | 0.80033,0.86693 270 | 0.4538,0.086235 271 | 0.43239,0.36644 272 | 0.82531,0.3692 273 | 0.08347,0.68503 274 | 0.13317,0.59794 275 | 0.17339,0.78936 276 | 0.39094,0.36765 277 | 0.83138,0.20603 278 | 0.80336,0.086667 279 | 0.060471,0.77193 280 | 0.39926,0.20567 281 | 0.52688,0.38827 282 | 0.4168,0.55178 283 | 0.65686,0.22895 284 | 0.62797,0.64194 285 | 0.29198,0.48448 286 | 0.43165,0.15185 287 | 0.015487,0.78193 288 | 0.98406,0.10061 289 | 0.16717,0.29407 290 | 0.10622,0.23737 291 | 0.37241,0.53087 292 | 0.19812,0.091499 293 | 0.48969,0.40532 294 | 0.33949,0.10485 295 | 0.95163,0.11228 296 | 0.92033,0.78443 297 | 0.052677,0.29157 298 | 0.73786,0.60353 299 | 0.26912,0.96442 300 | 0.42284,0.43248 301 | 0.54787,0.69475 302 | 0.94274,0.7581 303 | 0.41774,0.43264 304 | 0.98305,0.6555 305 | 0.30145,0.10976 306 | 0.7011,0.93376 307 | 0.66634,0.18746 308 | 0.53913,0.26618 309 | 0.69811,0.79783 310 | 0.66653,0.4876 311 | 0.17813,0.76896 312 | 0.12801,0.39601 313 | 0.99908,0.27294 314 | 0.17112,0.037235 315 | 0.032601,0.67329 316 | 0.5612,0.42956 317 | 0.88187,0.45174 318 | 0.66918,0.60986 319 | 0.19043,0.059403 320 | 0.36892,0.31581 321 | 0.46073,0.77272 322 | 0.98164,0.69643 323 | 0.1564,0.12533 324 | 0.85552,0.13015 325 | 0.64476,0.092352 326 | 0.37627,0.0078203 327 | 0.19092,0.42311 328 | 0.42825,0.65557 329 | 0.48202,0.72292 330 | 0.12061,0.53121 331 | 0.58951,0.10882 332 | 0.22619,0.63177 333 | 0.38462,0.1265 334 | 0.58299,0.1343 335 | 0.25181,0.098594 336 | 0.29044,0.14203 337 | 0.61709,0.16825 338 | 0.26528,0.19625 339 | 0.82438,0.31748 340 | 0.98266,0.31643 341 | 0.73025,0.21756 342 | 0.34388,0.25104 343 | 0.58407,0.89292 344 | 0.10777,0.70322 345 | 0.90631,0.55574 346 | 0.87965,0.18443 347 | 0.81776,0.21203 348 | 0.26073,0.077347 349 | 0.59436,0.9138 350 | 0.022513,0.70672 351 | 0.42526,0.55779 352 | 0.31272,0.31343 353 | 0.16148,0.1662 354 | 0.17877,0.6225 355 | 0.42289,0.98793 356 | 0.094229,0.17043 357 | 0.59852,0.25779 358 | 0.47092,0.3968 359 | 0.69595,0.073995 360 | 0.69989,0.6841 361 | 0.63853,0.40239 362 | 0.033604,0.98284 363 | 0.068806,0.40218 364 | 0.3196,0.62067 365 | 0.53086,0.15437 366 | 0.65445,0.38135 367 | 0.40762,0.16113 368 | 0.81998,0.75811 369 | 0.71836,0.87111 370 | 0.96865,0.35078 371 | 0.53133,0.68554 372 | 0.32515,0.29415 373 | 0.10563,0.53063 374 | 0.61096,0.83242 375 | 0.7788,0.59749 376 | 0.42345,0.33531 377 | 0.090823,0.29923 378 | 0.26647,0.45259 379 | 0.15366,0.42265 380 | 0.28101,0.35961 381 | 0.44009,0.55832 382 | 0.52714,0.74255 383 | 0.45742,0.42433 384 | 0.87537,0.42936 385 | 0.51805,0.12487 386 | 0.94362,0.024434 387 | 0.63771,0.29019 388 | 0.95769,0.31752 389 | 0.24071,0.65369 390 | 0.67612,0.95694 391 | 0.28906,0.93573 392 | 0.67181,0.45789 393 | 0.69514,0.24048 394 | 0.067993,0.7639 395 | 0.25479,0.75933 396 | 0.22404,0.74065 397 | 0.66783,0.74369 398 | 0.84439,0.10592 399 | 0.34446,0.68156 400 | 0.78052,0.46326 401 | 0.67533,0.21216 402 | 0.0067153,0.098519 403 | 0.60217,0.82357 404 | 0.38677,0.17501 405 | 0.91599,0.16357 406 | 0.0011511,0.66599 407 | 0.46245,0.89439 408 | 0.42435,0.51656 409 | 0.46092,0.7027 410 | 0.77016,0.15359 411 | 0.32247,0.95346 412 | 0.78474,0.54088 413 | 0.47136,0.67973 414 | 0.035763,0.036563 415 | 0.17587,0.8092 416 | 0.72176,0.74862 417 | 0.47349,0.12019 418 | 0.15272,0.52505 419 | 0.34112,0.32583 420 | 0.60739,0.54645 421 | 0.19175,0.39888 422 | 0.73843,0.41509 423 | 0.24285,0.18074 424 | 0.91742,0.25539 425 | 0.26906,0.020536 426 | 0.7655,0.92368 427 | 0.18866,0.6537 428 | 0.2875,0.93261 429 | 0.091113,0.16351 430 | 0.57621,0.9211 431 | 0.68336,0.79466 432 | 0.54659,0.57739 433 | 0.42573,0.44004 434 | 0.64444,0.25761 435 | 0.64762,0.75195 436 | 0.67902,0.22867 437 | 0.63579,0.064187 438 | 0.94517,0.76733 439 | 0.20893,0.6712 440 | 0.70928,0.71521 441 | 0.23623,0.64206 442 | 0.1194,0.41905 443 | 0.6073,0.39076 444 | 0.45014,0.81614 445 | 0.45873,0.31743 446 | 0.66194,0.81454 447 | 0.77029,0.78907 448 | 0.35022,0.85226 449 | 0.66201,0.50564 450 | 0.41616,0.63566 451 | 0.84193,0.95089 452 | 0.83292,0.44396 453 | 0.25644,0.060019 454 | 0.61346,0.86675 455 | 0.58225,0.63119 456 | 0.54074,0.35507 457 | 0.86994,0.997 458 | 0.26478,0.22417 459 | 0.31807,0.65245 460 | 0.11921,0.60499 461 | 0.93983,0.38725 462 | 0.64555,0.14219 463 | 0.47946,0.025135 464 | 0.63932,0.42111 465 | 0.54472,0.1841 466 | 0.64731,0.72578 467 | 0.54389,0.37036 468 | 0.72105,0.84156 469 | 0.5225,0.73423 470 | 0.9937,0.57103 471 | 0.21868,0.17686 472 | 0.1058,0.95738 473 | 0.1097,0.26532 474 | 0.063591,0.92458 475 | 0.40458,0.22377 476 | 0.44837,0.37356 477 | 0.36582,0.0875 478 | 0.7635,0.64012 479 | 0.6279,0.18062 480 | 0.77198,0.045051 481 | 0.93285,0.72317 482 | 0.97274,0.34744 483 | 0.19203,0.66062 484 | 0.13887,0.38387 485 | 0.69627,0.62735 486 | 0.09382,0.02165 487 | 0.5254,0.91057 488 | 0.53034,0.80056 489 | 0.86114,0.74585 490 | 0.48485,0.81311 491 | 0.39346,0.38331 492 | 0.67143,0.61728 493 | 0.74126,0.57549 494 | 0.52005,0.53005 495 | 0.34771,0.27507 496 | 0.15,0.24863 497 | 0.58609,0.45164 498 | 0.26215,0.22771 499 | 0.044454,0.80445 500 | 0.75493,0.9861 501 | 5.03,5.3709 502 | 5.5357,5.8909 503 | 5.0871,5.8564 504 | 5.8021,5.4024 505 | 5.9891,5.318 506 | 5.0669,5.6086 507 | 5.9394,5.9102 508 | 5.0182,5.9091 509 | 5.6838,5.5916 510 | 5.7837,5.3326 511 | 5.5341,5.8531 512 | 5.8854,5.4424 513 | 5.899,5.9044 514 | 5.6259,5.0332 515 | 5.1379,5.5324 516 | 5.2178,5.7165 517 | 5.1821,5.1793 518 | 5.0418,5.3365 519 | 5.1069,5.1877 520 | 5.6164,5.3219 521 | 5.9397,5.4039 522 | 5.3545,5.5486 523 | 5.4106,5.0487 524 | 5.9843,5.5527 525 | 5.9456,5.2748 526 | 5.6766,5.2415 527 | 5.9883,5.2431 528 | 5.7668,5.1542 529 | 5.3367,5.9564 530 | 5.6624,5.9357 531 | 5.2442,5.8187 532 | 5.2955,5.7283 533 | 5.6802,5.1758 534 | 5.5278,5.3604 535 | 5.4116,5.1888 536 | 5.6026,5.0012 537 | 5.7505,5.3164 538 | 5.5835,5.6996 539 | 5.5518,5.6253 540 | 5.5836,5.5431 541 | 5.5118,5.439 542 | 5.0826,5.2874 543 | 5.7196,5.5017 544 | 5.9962,5.7615 545 | 5.3545,5.7624 546 | 5.9713,5.5761 547 | 5.3464,5.7477 548 | 5.8865,5.6455 549 | 5.4547,5.1232 550 | 5.4134,5.5044 551 | 5.2177,5.3473 552 | 5.1257,5.0921 553 | 5.3089,5.1478 554 | 5.7261,5.1982 555 | 5.7829,5.6723 556 | 5.6938,5.4315 557 | 5.0098,5.6944 558 | 5.8432,5.2568 559 | 5.9223,5.0098 560 | 5.771,5.5323 561 | 5.0427,5.2794 562 | 5.3782,5.9462 563 | 5.7043,5.9064 564 | 5.7295,5.3927 565 | 5.2243,5.0249 566 | 5.2691,5.6714 567 | 5.673,5.8372 568 | 5.4775,5.9715 569 | 5.6237,5.0569 570 | 5.2364,5.4503 571 | 5.1771,5.5825 572 | 5.8296,5.6866 573 | 5.7669,5.7194 574 | 5.9345,5.65 575 | 5.1079,5.7269 576 | 5.1822,5.3738 577 | 5.0991,5.5816 578 | 5.4898,5.1161 579 | 5.1932,5.0577 580 | 5.8959,5.9798 581 | 5.0991,5.2848 582 | 5.0442,5.595 583 | 5.5573,5.9622 584 | 5.7725,5.1858 585 | 5.3119,5.193 586 | 5.179,5.3416 587 | 5.339,5.9329 588 | 5.2101,5.3907 589 | 5.5102,5.2732 590 | 5.9064,5.1519 591 | 5.6289,5.3971 592 | 5.1015,5.3747 593 | 5.3909,5.1311 594 | 5.0546,5.435 595 | 5.5013,5.0915 596 | 5.4317,5.6146 597 | 5.9976,5.011 598 | 5.8116,5.5733 599 | 5.4857,5.7897 600 | 5.8944,5.2354 601 | 5.1375,5.448 602 | 5.39,5.5694 603 | 5.9274,5.0614 604 | 5.9175,5.4963 605 | 5.7136,5.6423 606 | 5.6183,5.2213 607 | 5.3433,5.8371 608 | 5.936,5.9711 609 | 5.1248,5.8464 610 | 5.7306,5.506 611 | 5.6465,5.2789 612 | 5.8332,5.7466 613 | 5.3983,5.2369 614 | 5.7498,5.9573 615 | 5.8352,5.6203 616 | 5.3225,5.6003 617 | 5.5523,5.1726 618 | 5.9791,5.0903 619 | 5.5493,5.2553 620 | 5.3304,5.8586 621 | 5.6195,5.9111 622 | 5.3606,5.6996 623 | 5.7565,5.7252 624 | 5.4139,5.2299 625 | 5.4923,5.5761 626 | 5.6947,5.8106 627 | 5.9727,5.4038 628 | 5.3278,5.9884 629 | 5.8378,5.09 630 | 5.7391,5.3209 631 | 5.9542,5.5114 632 | 5.0319,5.0606 633 | 5.3569,5.7257 634 | 5.6627,5.5566 635 | 5.2815,5.5294 636 | 5.2304,5.83 637 | 5.7111,5.8588 638 | 5.6246,5.789 639 | 5.5906,5.3178 640 | 5.6604,5.4522 641 | 5.0476,5.7522 642 | 5.3488,5.1099 643 | 5.4513,5.1097 644 | 5.2409,5.2699 645 | 5.715,5.5246 646 | 5.8562,5.9727 647 | 5.2815,5.7104 648 | 5.7311,5.3119 649 | 5.1378,5.2915 650 | 5.8367,5.8504 651 | 5.1386,5.9116 652 | 5.5882,5.6393 653 | 5.3662,5.2554 654 | 5.8068,5.0887 655 | 5.5038,5.8383 656 | 5.4896,5.5847 657 | 5.877,5.9481 658 | 5.3531,5.061 659 | 5.4494,5.5846 660 | 5.9635,5.2851 661 | 5.0423,5.8277 662 | 5.973,5.191 663 | 5.1892,5.4425 664 | 5.6671,5.3934 665 | 5.5864,5.8266 666 | 5.6751,5.6769 667 | 5.361,5.2076 668 | 5.6203,5.3181 669 | 5.8112,5.1338 670 | 5.0193,5.6715 671 | 5.0839,5.571 672 | 5.9748,5.1698 673 | 5.6513,5.1477 674 | 5.2312,5.4761 675 | 5.4035,5.9081 676 | 5.122,5.5522 677 | 5.2684,5.0329 678 | 5.2578,5.0539 679 | 5.3317,5.8051 680 | 5.1522,5.4514 681 | 5.348,5.3826 682 | 5.1217,5.7896 683 | 5.8842,5.3643 684 | 5.0943,5.5323 685 | 5.93,5.7117 686 | 5.399,5.8715 687 | 5.0474,5.3287 688 | 5.3424,5.6501 689 | 5.736,5.9748 690 | 5.7947,5.076 691 | 5.5449,5.587 692 | 5.6862,5.4139 693 | 5.8936,5.3091 694 | 5.0548,5.2638 695 | 5.3037,5.7588 696 | 5.0462,5.9952 697 | 5.1955,5.1866 698 | 5.7202,5.7811 699 | 5.7218,5.1958 700 | 5.8778,5.9924 701 | 5.5824,5.8023 702 | 5.0707,5.4242 703 | 5.9227,5.7289 704 | 5.8004,5.4984 705 | 5.2859,5.809 706 | 5.5437,5.3565 707 | 5.9848,5.0732 708 | 5.7157,5.591 709 | 5.839,5.9102 710 | 5.4333,5.1938 711 | 5.4706,5.4324 712 | 5.5607,5.7492 713 | 5.2691,5.0392 714 | 5.749,5.9463 715 | 5.5039,5.7637 716 | 5.6468,5.5588 717 | 5.3077,5.1838 718 | 5.1387,5.4979 719 | 5.4756,5.5178 720 | 5.3625,5.9942 721 | 5.7881,5.8549 722 | 5.7803,5.9624 723 | 5.6685,5.6789 724 | 5.1335,5.4035 725 | 5.0216,5.935 726 | 5.5598,5.4795 727 | 5.3008,5.2318 728 | 5.9394,5.3963 729 | 5.9809,5.7051 730 | 5.2866,5.5586 731 | 5.8008,5.7566 732 | 5.8961,5.9955 733 | 5.5975,5.9624 734 | 5.884,5.5351 735 | 5.9437,5.9639 736 | 5.5492,5.1156 737 | 5.7284,5.0514 738 | 5.5768,5.3043 739 | 5.0259,5.5802 740 | 5.4465,5.531 741 | 5.6463,5.9012 742 | 5.5212,5.5406 743 | 5.3723,5.432 744 | 5.9371,5.5427 745 | 5.8295,5.7124 746 | 5.8491,5.0167 747 | 5.3725,5.8009 748 | 5.5932,5.1425 749 | 5.8726,5.4785 750 | 5.9335,5.2568 751 | 5.6685,5.3691 752 | 5.2068,5.6618 753 | 5.6539,5.1696 754 | 5.0721,5.2788 755 | 5.4067,5.1982 756 | 5.6669,5.1951 757 | 5.9337,5.3268 758 | 5.811,5.8803 759 | 5.4845,5.4711 760 | 5.7567,5.404 761 | 5.417,5.1792 762 | 5.9718,5.9689 763 | 5.988,5.4075 764 | 5.8641,5.8445 765 | 5.3889,5.6153 766 | 5.4547,5.3766 767 | 5.2467,5.8772 768 | 5.7844,5.7849 769 | 5.8828,5.465 770 | 5.9137,5.814 771 | 5.5583,5.8984 772 | 5.5989,5.4292 773 | 5.1489,5.3343 774 | 5.8997,5.5966 775 | 5.4504,5.902 776 | 5.2057,5.7021 777 | 5.8997,5.3775 778 | 5.7626,5.735 779 | 5.8825,5.9541 780 | 5.285,5.5428 781 | 5.6732,5.5401 782 | 5.6643,5.3111 783 | 5.1228,5.0712 784 | 5.4073,5.182 785 | 5.2753,5.093 786 | 5.7167,5.4635 787 | 5.2834,5.0093 788 | 5.8962,5.915 789 | 5.8266,5.6427 790 | 5.39,5.0014 791 | 5.4979,5.0304 792 | 5.6948,5.2085 793 | 5.8344,5.455 794 | 5.6096,5.1273 795 | 5.5747,5.0086 796 | 5.326,5.7271 797 | 5.4564,5.3541 798 | 5.7138,5.7804 799 | 5.8844,5.4367 800 | 5.7209,5.4366 801 | 5.0186,5.0492 802 | 5.6748,5.0496 803 | 5.4385,5.0911 804 | 5.4378,5.594 805 | 5.117,5.2411 806 | 5.8147,5.8414 807 | 5.3249,5.8572 808 | 5.2462,5.9636 809 | 5.3427,5.4889 810 | 5.3757,5.2203 811 | 5.5466,5.2262 812 | 5.5619,5.5368 813 | 5.3958,5.7621 814 | 5.3981,5.3476 815 | 5.5154,5.4612 816 | 5.6575,5.6393 817 | 5.9509,5.9173 818 | 5.7223,5.1616 819 | 5.4001,5.7156 820 | 5.8319,5.5777 821 | 5.1343,5.4333 822 | 5.0605,5.8842 823 | 5.0842,5.3931 824 | 5.1639,5.179 825 | 5.3242,5.6333 826 | 5.3017,5.624 827 | 5.0117,5.3279 828 | 5.5399,5.803 829 | 5.0954,5.9995 830 | 5.1465,5.981 831 | 5.6311,5.127 832 | 5.8593,5.2322 833 | 5.9742,5.0236 834 | 5.5708,5.6074 835 | 5.9969,5.1108 836 | 5.5535,5.4075 837 | 5.5155,5.8841 838 | 5.3307,5.5481 839 | 5.43,5.369 840 | 5.4918,5.2083 841 | 5.071,5.4409 842 | 5.8877,5.9562 843 | 5.0646,5.124 844 | 5.4362,5.4708 845 | 5.8266,5.8569 846 | 5.3945,5.0434 847 | 5.6135,5.6916 848 | 5.8186,5.979 849 | 5.8862,5.2833 850 | 5.9311,5.1338 851 | 5.1908,5.6853 852 | 5.2586,5.9095 853 | 5.8979,5.6109 854 | 5.5934,5.9 855 | 5.5038,5.1934 856 | 5.6128,5.7544 857 | 5.8194,5.3463 858 | 5.5319,5.4186 859 | 5.2021,5.1557 860 | 5.4539,5.819 861 | 5.4279,5.6249 862 | 5.9661,5.7386 863 | 5.6201,5.8051 864 | 5.6954,5.0672 865 | 5.7202,5.9508 866 | 5.3469,5.4976 867 | 5.517,5.7551 868 | 5.5567,5.7424 869 | 5.1565,5.8311 870 | 5.5621,5.1565 871 | 5.6948,5.4573 872 | 5.4265,5.6181 873 | 5.8363,5.9322 874 | 5.7314,5.8351 875 | 5.36,5.8954 876 | 5.4542,5.5825 877 | 5.3864,5.5827 878 | 5.7756,5.8549 879 | 5.7343,5.0349 880 | 5.4303,5.8854 881 | 5.6938,5.4077 882 | 5.9452,5.0364 883 | 5.7842,5.7461 884 | 5.7056,5.1548 885 | 5.1093,5.1439 886 | 5.3899,5.606 887 | 5.5909,5.2545 888 | 5.4594,5.3242 889 | 5.0503,5.4018 890 | 5.2287,5.4064 891 | 5.8342,5.3862 892 | 5.0156,5.6098 893 | 5.8637,5.1669 894 | 5.0781,5.1881 895 | 5.669,5.0946 896 | 5.5002,5.3232 897 | 5.218,5.7696 898 | 5.5716,5.2341 899 | 5.1222,5.7404 900 | 5.6712,5.6928 901 | 5.5996,5.8241 902 | 5.056,5.828 903 | 5.0563,5.2934 904 | 5.1525,5.3094 905 | 5.0196,5.523 906 | 5.4352,5.3253 907 | 5.8322,5.8318 908 | 5.6174,5.8103 909 | 5.5201,5.557 910 | 5.8639,5.263 911 | 5.0977,5.6806 912 | 5.9081,5.2337 913 | 5.108,5.4564 914 | 5.517,5.3846 915 | 5.1432,5.5386 916 | 5.5594,5.9917 917 | 5.0046,5.7552 918 | 5.7667,5.9805 919 | 5.8487,5.2348 920 | 5.9168,5.5286 921 | 5.987,5.0514 922 | 5.5051,5.7569 923 | 5.2714,5.602 924 | 5.1008,5.8572 925 | 5.5078,5.9883 926 | 5.5856,5.9295 927 | 5.7629,5.4095 928 | 5.083,5.0003 929 | 5.6616,5.5409 930 | 5.517,5.2077 931 | 5.171,5.2193 932 | 5.9386,5.3258 933 | 5.5905,5.0959 934 | 5.4406,5.7475 935 | 5.9419,5.7485 936 | 5.6559,5.5433 937 | 5.4519,5.3381 938 | 5.8397,5.8323 939 | 5.5326,5.5526 940 | 5.5539,5.9575 941 | 5.6801,5.8928 942 | 5.3672,5.3565 943 | 5.2393,5.5464 944 | 5.5789,5.3467 945 | 5.8669,5.6228 946 | 5.4068,5.7966 947 | 5.1126,5.7459 948 | 5.4438,5.1255 949 | 5.3002,5.8224 950 | 5.4014,5.0252 951 | 5.8334,5.4144 952 | 5.4036,5.7314 953 | 5.3902,5.7814 954 | 5.3604,5.3673 955 | 5.1403,5.7449 956 | 5.2601,5.8923 957 | 5.0868,5.2426 958 | 5.4294,5.1296 959 | 5.2573,5.2251 960 | 5.2976,5.35 961 | 5.4249,5.2871 962 | 5.1192,5.9275 963 | 5.4951,5.0513 964 | 5.7064,5.5927 965 | 5.2436,5.1629 966 | 5.7851,5.8384 967 | 5.0741,5.1676 968 | 5.3939,5.5022 969 | 5.0034,5.9993 970 | 5.2207,5.3554 971 | 5.0013,5.0471 972 | 5.1892,5.2137 973 | 5.1425,5.3978 974 | 5.2681,5.3337 975 | 5.1749,5.2296 976 | 5.1386,5.9361 977 | 5.5989,5.6832 978 | 5.9011,5.9621 979 | 5.9394,5.438 980 | 5.2212,5.9403 981 | 5.4827,5.0058 982 | 5.376,5.6103 983 | 5.5238,5.8011 984 | 5.2649,5.233 985 | 5.0684,5.9325 986 | 5.4363,5.7633 987 | 5.1739,5.8264 988 | 5.0261,5.5735 989 | 5.9547,5.7926 990 | 5.4306,5.329 991 | 5.9616,5.2235 992 | 5.7624,5.3124 993 | 5.0073,5.5845 994 | 5.68,5.8299 995 | 5.706,5.2905 996 | 5.6451,5.4026 997 | 5.5523,5.8621 998 | 5.2181,5.6147 999 | 5.7724,5.9912 1000 | 5.228,5.2037 1001 | 2.4889,4.8561 1002 | 2.9714,4.4984 1003 | 2.1125,4.0488 1004 | 2.7432,4.3138 1005 | 2.6385,4.6416 1006 | 2.5942,4.7864 1007 | 2.4986,4.2892 1008 | 2.5679,4.4979 1009 | 2.4265,4.8184 1010 | 2.0762,4.5951 1011 | 2.2906,4.5364 1012 | 2.5613,4.3309 1013 | 2.6333,4.4117 1014 | 2.9308,4.794 1015 | 2.9778,4.3432 1016 | 2.0936,4.4626 1017 | 2.6617,4.3678 1018 | 2.6028,4.6796 1019 | 2.4738,4.5678 1020 | 2.3563,4.6518 1021 | 2.4756,4.4911 1022 | 2.671,4.3985 1023 | 2.9596,4.4775 1024 | 2.0891,4.0666 1025 | 2.7977,4.411 1026 | 2.5908,4.9691 1027 | 2.9122,4.7807 1028 | 2.1011,4.729 1029 | 2.2933,4.7657 1030 | 2.0516,4.7566 1031 | 2.5041,4.8433 1032 | 2.7684,4.7702 1033 | 2.283,4.9787 1034 | 2.2254,4.1114 1035 | 2.3313,4.3961 1036 | 2.4533,4.4921 1037 | 2.7374,4.2581 1038 | 2.5099,4.037 1039 | 2.3825,4.9744 1040 | 2.9055,4.7264 1041 | 2.9653,4.148 1042 | 2.6283,4.1479 1043 | 2.132,4.7048 1044 | 2.6183,4.381 1045 | 2.383,4.0764 1046 | 2.9912,4.4108 1047 | 2.2868,4.143 1048 | 2.7062,4.7989 1049 | 2.5352,4.9302 1050 | 2.1932,4.0047 1051 | 2.6894,4.65 1052 | 2.0505,4.6785 1053 | 2.1844,4.2536 1054 | 2.0457,4.8432 1055 | 2.885,4.294 1056 | 2.8398,4.0269 1057 | 2.1182,4.0933 1058 | 2.4104,4.7979 1059 | 2.1202,4.7114 1060 | 2.5721,4.7834 1061 | 2.9494,4.6239 1062 | 2.2564,4.8254 1063 | 2.9899,4.035 1064 | 2.3498,4.4055 1065 | 2.2085,4.2497 1066 | 2.6658,4.4809 1067 | 2.9733,4.8808 1068 | 2.6227,4.2807 1069 | 2.0635,4.5991 1070 | 2.3735,4.0262 1071 | 2.1663,4.1552 1072 | 2.2313,4.8339 1073 | 2.0522,4.1949 1074 | 2.9018,4.8298 1075 | 2.7933,4.3381 1076 | 2.373,4.6711 1077 | 2.8321,4.0524 1078 | 2.7538,4.7343 1079 | 2.6219,4.4995 1080 | 2.3941,4.9433 1081 | 2.3593,4.2898 1082 | 2.0889,4.3766 1083 | 2.3417,4.1138 1084 | 2.5487,4.9649 1085 | 2.4605,4.4325 1086 | 2.6455,4.0846 1087 | 2.5135,4.7167 1088 | 2.8144,4.5068 1089 | 2.0972,4.3281 1090 | 2.4637,4.7535 1091 | 2.5898,4.836 1092 | 2.1872,4.2537 1093 | 2.6113,4.5344 1094 | 2.0519,4.4352 1095 | 2.5757,4.1577 1096 | 2.8423,4.6005 1097 | 2.4997,4.9375 1098 | 2.439,4.1078 1099 | 2.1491,4.9 1100 | 2.0283,4.5505 1101 | 2.7567,4.4274 1102 | 2.7961,4.1524 1103 | 2.2936,4.2475 1104 | 2.1152,4.4474 1105 | 2.3751,4.5328 1106 | 2.8289,4.3547 1107 | 2.8418,4.7731 1108 | 2.6652,4.8817 1109 | 2.9601,4.7341 1110 | 2.9431,4.4064 1111 | 2.1127,4.6042 1112 | 2.6483,4.6411 1113 | 2.4808,4.1275 1114 | 2.0665,4.4962 1115 | 2.8978,4.3105 1116 | 2.4972,4.5786 1117 | 2.7713,4.9436 1118 | 2.0604,4.4269 1119 | 2.2625,4.0331 1120 | 2.6511,4.9294 1121 | 2.1336,4.925 1122 | 2.6385,4.3583 1123 | 2.3849,4.26 1124 | 2.7657,4.7869 1125 | 2.6529,4.5116 1126 | 2.3815,4.5625 1127 | 2.3,4.6848 1128 | 2.3401,4.0924 1129 | 2.9189,4.8726 1130 | 2.4563,4.9429 1131 | 2.4425,4.0966 1132 | 2.4542,4.8459 1133 | 2.9453,4.9094 1134 | 2.2191,4.0113 1135 | 2.8824,4.5237 1136 | 2.0199,4.6503 1137 | 2.3418,4.3851 1138 | 2.766,4.6493 1139 | 2.3428,4.7629 1140 | 2.6188,4.5757 1141 | 2.453,4.6319 1142 | 2.0102,4.2782 1143 | 2.5991,4.8398 1144 | 2.6016,4.4268 1145 | 2.6494,4.6316 1146 | 2.3427,4.8335 1147 | 2.4933,4.2702 1148 | 2.7018,4.4008 1149 | 2.8878,4.5543 1150 | 2.0551,4.4439 1151 | 2.0984,4.0904 1152 | 2.6498,4.7444 1153 | 2.7641,4.0326 1154 | 2.988,4.4297 1155 | 2.1253,4.0373 1156 | 2.3645,4.9758 1157 | 2.6762,4.5223 1158 | 2.3758,4.9096 1159 | 2.8635,4.3832 1160 | 2.292,4.8845 1161 | 2.1335,4.255 1162 | 2.6727,4.909 1163 | 2.2026,4.8946 1164 | 2.8685,4.3985 1165 | 2.7512,4.625 1166 | 2.4194,4.5676 1167 | 2.0002,4.8945 1168 | 2.1495,4.2142 1169 | 2.2738,4.0039 1170 | 2.8724,4.8806 1171 | 2.6013,4.2351 1172 | 2.3212,4.2449 1173 | 2.2843,4.6409 1174 | 2.4353,4.3045 1175 | 2.9038,4.8256 1176 | 2.9251,4.8837 1177 | 2.5053,4.9454 1178 | 2.6276,4.3908 1179 | 2.7193,4.8013 1180 | 2.0239,4.1571 1181 | 2.5749,4.6252 1182 | 2.0465,4.699 1183 | 2.4225,4.0859 1184 | 2.4677,4.5312 1185 | 2.0226,4.8886 1186 | 2.0651,4.2637 1187 | 2.924,4.2348 1188 | 2.5341,4.8397 1189 | 2.3668,4.4955 1190 | 2.3639,4.1524 1191 | 2.1514,4.2308 1192 | 2.1496,4.658 1193 | 2.3508,4.5629 1194 | 2.336,4.2918 1195 | 2.784,4.6223 1196 | 2.4867,4.7159 1197 | 2.4648,4.2807 1198 | 2.1313,4.4123 1199 | 2.8864,4.3622 1200 | 2.6746,4.7814 1201 | 2.8352,4.1355 1202 | 2.6565,4.9021 1203 | 2.9839,4.2896 1204 | 2.9798,4.4996 1205 | 2.2502,4.7836 1206 | 2.6246,4.6771 1207 | 2.7282,4.1498 1208 | 2.4982,4.6966 1209 | 2.8498,4.129 1210 | 2.1909,4.9459 1211 | 2.1241,4.8864 1212 | 2.0028,4.515 1213 | 2.153,4.6794 1214 | 2.5342,4.9768 1215 | 2.5106,4.1255 1216 | 2.3852,4.7522 1217 | 2.3106,4.8271 1218 | 2.0036,4.7814 1219 | 2.8152,4.1909 1220 | 2.6384,4.4286 1221 | 2.4483,4.0145 1222 | 2.2441,4.3253 1223 | 2.8034,4.1347 1224 | 2.824,4.4505 1225 | 2.8522,4.5723 1226 | 2.4673,4.792 1227 | 2.9707,4.4197 1228 | 2.8412,4.5325 1229 | 2.0785,4.9257 1230 | 2.2376,4.8991 1231 | 2.8176,4.5448 1232 | 2.4058,4.9011 1233 | 2.4663,4.0518 1234 | 2.9515,4.8086 1235 | 2.965,4.3349 1236 | 2.7653,4.2287 1237 | 2.5745,4.8224 1238 | 2.9159,4.3482 1239 | 2.4954,4.1655 1240 | 2.166,4.0281 1241 | 2.326,4.9554 1242 | 2.2964,4.6803 1243 | 2.5583,4.8606 1244 | 2.0675,4.9391 1245 | 2.069,4.6802 1246 | 2.1668,4.9174 1247 | 2.9474,4.2567 1248 | 2.8111,4.8856 1249 | 2.7105,4.92 1250 | 2.9702,4.3001 1251 | 2.9984,4.0734 1252 | 2.9875,4.7674 1253 | 2.1501,4.085 1254 | 2.9585,4.7288 1255 | 2.5305,4.4479 1256 | 2.0741,4.6512 1257 | 2.3118,4.1695 1258 | 2.8952,4.5314 1259 | 2.8348,4.6338 1260 | 2.0023,4.0141 1261 | 2.6402,4.4704 1262 | 2.8032,4.8863 1263 | 2.2451,4.114 1264 | 2.0641,4.4425 1265 | 2.2631,4.6595 1266 | 2.1027,4.2948 1267 | 2.4837,4.9504 1268 | 2.4189,4.6943 1269 | 2.3813,4.2068 1270 | 2.8868,4.5548 1271 | 2.4206,4.8793 1272 | 2.2838,4.5579 1273 | 2.0482,4.7523 1274 | 2.2192,4.8949 1275 | 2.2392,4.8418 1276 | 2.0293,4.1309 1277 | 2.7023,4.1892 1278 | 2.0076,4.1536 1279 | 2.6109,4.0289 1280 | 2.4081,4.0091 1281 | 2.2489,4.5965 1282 | 2.6525,4.609 1283 | 2.3203,4.9189 1284 | 2.1037,4.7336 1285 | 2.5356,4.3011 1286 | 2.1649,4.4956 1287 | 2.8834,4.2582 1288 | 2.6665,4.7329 1289 | 2.8477,4.1168 1290 | 2.7627,4.746 1291 | 2.807,4.8098 1292 | 2.633,4.7452 1293 | 2.7104,4.3371 1294 | 2.6887,4.5843 1295 | 2.3209,4.469 1296 | 2.5316,4.0873 1297 | 2.8732,4.8287 1298 | 2.0545,4.6859 1299 | 2.5004,4.2673 1300 | 2.4328,4.9695 1301 | 2.9043,4.1838 1302 | 2.6302,4.2999 1303 | 2.983,4.4112 1304 | 2.5852,4.2365 1305 | 2.8406,4.1951 1306 | 2.4688,4.7054 1307 | 2.5452,4.1805 1308 | 2.1791,4.5223 1309 | 2.6345,4.2962 1310 | 2.963,4.4628 1311 | 2.534,4.9252 1312 | 2.4796,4.2159 1313 | 2.7937,4.001 1314 | 2.0927,4.9066 1315 | 2.8808,4.68 1316 | 2.0039,4.515 1317 | 2.5115,4.5221 1318 | 2.6785,4.1029 1319 | 2.5657,4.9969 1320 | 2.4785,4.359 1321 | 2.3205,4.6252 1322 | 2.6016,4.3934 1323 | 2.9132,4.0077 1324 | 2.6825,4.5453 1325 | 2.9467,4.5091 1326 | 2.0991,4.2468 1327 | 2.511,4.0454 1328 | 2.1101,4.8417 1329 | 2.5453,4.0482 1330 | 2.6888,4.3163 1331 | 2.1474,4.7834 1332 | 2.7776,4.9724 1333 | 2.3991,4.5865 1334 | 2.8983,4.778 1335 | 2.307,4.7277 1336 | 2.0611,4.651 1337 | 2.2195,4.6646 1338 | 2.0828,4.9388 1339 | 2.9504,4.5351 1340 | 2.0164,4.3984 1341 | 2.1147,4.6705 1342 | 2.0124,4.4405 1343 | 2.2162,4.1329 1344 | 2.0114,4.4392 1345 | 2.6424,4.5476 1346 | 2.517,4.3951 1347 | 2.2455,4.3983 1348 | 2.1937,4.7513 1349 | 2.0909,4.5224 1350 | 2.3684,4.4904 1351 | 2.0078,4.0887 1352 | 2.6027,4.2509 1353 | 2.4789,4.4476 1354 | 2.3081,4.638 1355 | 2.7444,4.7094 1356 | 2.8393,4.9926 1357 | 2.2624,4.9322 1358 | 2.5142,4.0922 1359 | 2.4468,4.9535 1360 | 2.3412,4.1628 1361 | 2.8391,4.9705 1362 | 2.9825,4.597 1363 | 2.6265,4.2402 1364 | 2.1813,4.0703 1365 | 2.123,4.3 1366 | 2.58,4.8135 1367 | 2.3285,4.0767 1368 | 2.2682,4.3545 1369 | 2.5502,4.132 1370 | 2.1805,4.1582 1371 | 2.6785,4.0621 1372 | 2.0557,4.7018 1373 | 2.0341,4.0865 1374 | 2.2865,4.6168 1375 | 2.0774,4.1738 1376 | 2.9006,4.6514 1377 | 2.8466,4.4987 1378 | 2.3957,4.2845 1379 | 2.1692,4.8306 1380 | 2.4305,4.8184 1381 | 2.4162,4.9382 1382 | 2.7288,4.0003 1383 | 2.4065,4.6404 1384 | 2.9518,4.0074 1385 | 2.912,4.1064 1386 | 2.9514,4.1068 1387 | 2.346,4.3671 1388 | 2.2902,4.2396 1389 | 2.8867,4.3461 1390 | 2.21,4.2496 1391 | 2.1309,4.3871 1392 | 2.5205,4.421 1393 | 2.9055,4.6401 1394 | 2.4025,4.7876 1395 | 2.2158,4.27 1396 | 2.0787,4.844 1397 | 2.9331,4.7405 1398 | 2.6029,4.8261 1399 | 2.3775,4.1822 1400 | 2.6649,4.0654 1401 | 2.7922,4.6104 1402 | 2.3335,4.7016 1403 | 2.6927,4.1116 1404 | 2.2038,4.0958 1405 | 2.9587,4.5978 1406 | 2.7118,4.8122 1407 | 2.1669,4.8146 1408 | 2.4428,4.0894 1409 | 2.633,4.7313 1410 | 2.93,4.9039 1411 | 2.5293,4.4522 1412 | 2.6265,4.0707 1413 | 2.6808,4.2413 1414 | 2.9232,4.7319 1415 | 2.1528,4.0405 1416 | 2.4057,4.4245 1417 | 2.3125,4.5402 1418 | 2.6939,4.9538 1419 | 2.8907,4.2089 1420 | 2.4907,4.1163 1421 | 2.8058,4.6462 1422 | 2.3264,4.1084 1423 | 2.5499,4.9835 1424 | 2.3888,4.2483 1425 | 2.8968,4.6064 1426 | 2.6761,4.8167 1427 | 2.8284,4.8301 1428 | 2.1101,4.489 1429 | 2.2792,4.7607 1430 | 2.7676,4.9151 1431 | 2.2161,4.901 1432 | 2.0341,4.2142 1433 | 2.4366,4.5471 1434 | 2.9369,4.7847 1435 | 2.2621,4.1944 1436 | 2.5697,4.7469 1437 | 2.3596,4.4756 1438 | 2.0268,4.5833 1439 | 2.5004,4.2605 1440 | 2.827,4.0848 1441 | 2.259,4.2981 1442 | 2.0459,4.9171 1443 | 2.2465,4.4705 1444 | 2.6607,4.2695 1445 | 2.3294,4.763 1446 | 2.6595,4.7722 1447 | 2.013,4.0213 1448 | 2.7181,4.88 1449 | 2.3911,4.7982 1450 | 2.0335,4.3242 1451 | 2.406,4.669 1452 | 2.7163,4.2963 1453 | 2.9213,4.93 1454 | 2.984,4.282 1455 | 2.9834,4.1689 1456 | 2.8963,4.7452 1457 | 2.8657,4.4771 1458 | 2.801,4.6534 1459 | 2.555,4.9666 1460 | 2.4189,4.313 1461 | 2.1271,4.0764 1462 | 2.6546,4.7914 1463 | 2.864,4.3654 1464 | 2.2746,4.5851 1465 | 2.8402,4.1833 1466 | 2.0708,4.0769 1467 | 2.3788,4.1537 1468 | 2.2682,4.8269 1469 | 2.1529,4.301 1470 | 2.631,4.3839 1471 | 2.3164,4.6507 1472 | 2.9591,4.8174 1473 | 2.4987,4.7663 1474 | 2.7386,4.3742 1475 | 2.0128,4.1899 1476 | 2.6054,4.6465 1477 | 2.5765,4.0036 1478 | 2.8074,4.2829 1479 | 2.655,4.6386 1480 | 2.8782,4.5921 1481 | 2.9024,4.3253 1482 | 2.1522,4.989 1483 | 2.1926,4.1232 1484 | 2.791,4.7359 1485 | 2.0607,4.1566 1486 | 2.3898,4.4346 1487 | 2.3,4.8322 1488 | 2.7342,4.3599 1489 | 2.1042,4.0762 1490 | 2.7926,4.5569 1491 | 2.7827,4.2739 1492 | 2.5324,4.1321 1493 | 2.2534,4.6997 1494 | 2.071,4.4859 1495 | 2.6258,4.1827 1496 | 2.0247,4.1012 1497 | 2.062,4.2016 1498 | 2.1296,4.1347 1499 | 2.4506,4.3238 1500 | 2.6723,4.9505 1501 | 3.5321,2.6306 1502 | 3.2477,2.0142 1503 | 3.4373,2.3165 1504 | 3.6691,2.1119 1505 | 3.5477,2.6295 1506 | 3.6091,2.0607 1507 | 3.8631,2.674 1508 | 3.3807,2.4774 1509 | 3.749,2.3055 1510 | 3.1567,2.5163 1511 | 3.0581,2.707 1512 | 3.3397,2.8136 1513 | 3.8172,2.3158 1514 | 3.3775,2.3113 1515 | 3.9726,2.345 1516 | 3.6053,2.6663 1517 | 3.3382,2.8611 1518 | 3.928,2.7618 1519 | 3.8984,2.8758 1520 | 3.8507,2.8712 1521 | 3.2568,2.1728 1522 | 3.2855,2.8502 1523 | 3.7799,2.9596 1524 | 3.7014,2.7702 1525 | 3.4925,2.875 1526 | 3.9677,2.0674 1527 | 3.4762,2.6468 1528 | 3.9949,2.3241 1529 | 3.4906,2.6403 1530 | 3.5035,2.8798 1531 | 3.7688,2.3736 1532 | 3.3881,2.7667 1533 | 3.4533,2.1681 1534 | 3.1329,2.5197 1535 | 3.7585,2.6275 1536 | 3.5652,2.7139 1537 | 3.6486,2.3064 1538 | 3.7981,2.2637 1539 | 3.2204,2.916 1540 | 3.8579,2.615 1541 | 3.9047,2.0932 1542 | 3.292,2.6277 1543 | 3.7259,2.192 1544 | 3.3394,2.777 1545 | 3.2727,2.8645 1546 | 3.1703,2.3336 1547 | 3.664,2.1354 1548 | 3.5359,2.7655 1549 | 3.8291,2.3186 1550 | 3.2674,2.2524 1551 | 3.1762,2.2001 1552 | 3.4312,2.069 1553 | 3.4757,2.5519 1554 | 3.7852,2.4038 1555 | 3.1307,2.7501 1556 | 3.0514,2.4872 1557 | 3.6275,2.3848 1558 | 3.0291,2.0614 1559 | 3.1362,2.2137 1560 | 3.6946,2.5439 1561 | 3.5157,2.4106 1562 | 3.5426,2.901 1563 | 3.8085,2.0563 1564 | 3.7937,2.4435 1565 | 3.5019,2.5378 1566 | 3.2766,2.1341 1567 | 3.1197,2.5409 1568 | 3.8866,2.8574 1569 | 3.9703,2.198 1570 | 3.9425,2.1556 1571 | 3.6381,2.0614 1572 | 3.0906,2.6611 1573 | 3.0747,2.0186 1574 | 3.1825,2.2911 1575 | 3.0317,2.9738 1576 | 3.7249,2.7646 1577 | 3.1442,2.2437 1578 | 3.6359,2.6821 1579 | 3.7898,2.1379 1580 | 3.5663,2.6298 1581 | 3.3774,2.857 1582 | 3.8216,2.8998 1583 | 3.3049,2.3484 1584 | 3.3194,2.4863 1585 | 3.785,2.6795 1586 | 3.5037,2.7041 1587 | 3.261,2.4609 1588 | 3.7325,2.3643 1589 | 3.1629,2.2803 1590 | 3.9211,2.0762 1591 | 3.2222,2.4446 1592 | 3.0836,2.1657 1593 | 3.0737,2.3987 1594 | 3.7696,2.9206 1595 | 3.8177,2.5113 1596 | 3.7404,2.9141 1597 | 3.7582,2.0919 1598 | 3.9612,2.993 1599 | 3.4664,2.0964 1600 | 3.787,2.3131 1601 | 3.4226,2.7854 1602 | 3.9437,2.6024 1603 | 3.0013,2.4659 1604 | 3.9813,2.2981 1605 | 3.5702,2.1332 1606 | 3.3465,2.295 1607 | 3.5575,2.1666 1608 | 3.2998,2.3171 1609 | 3.1591,2.1098 1610 | 3.6653,2.8321 1611 | 3.6842,2.9716 1612 | 3.7924,2.2183 1613 | 3.3486,2.7061 1614 | 3.2501,2.039 1615 | 3.345,2.6163 1616 | 3.3286,2.6694 1617 | 3.9275,2.0372 1618 | 3.7561,2.0033 1619 | 3.2882,2.1425 1620 | 3.6062,2.8624 1621 | 3.766,2.276 1622 | 3.8462,2.5317 1623 | 3.902,2.5222 1624 | 3.5957,2.5676 1625 | 3.0685,2.333 1626 | 3.218,2.4134 1627 | 3.8694,2.4143 1628 | 3.4142,2.9839 1629 | 3.6612,2.0577 1630 | 3.7835,2.3965 1631 | 3.2479,2.7913 1632 | 3.5544,2.5942 1633 | 3.2296,2.3096 1634 | 3.0069,2.9018 1635 | 3.7666,2.0931 1636 | 3.0218,2.319 1637 | 3.3931,2.887 1638 | 3.2525,2.6574 1639 | 3.2042,2.6845 1640 | 3.6623,2.4739 1641 | 3.9147,2.1412 1642 | 3.0069,2.9509 1643 | 3.7464,2.8826 1644 | 3.7997,2.4374 1645 | 3.9078,2.835 1646 | 3.9746,2.3251 1647 | 3.1199,2.3676 1648 | 3.519,2.7948 1649 | 3.822,2.0993 1650 | 3.637,2.9518 1651 | 3.9539,2.0015 1652 | 3.9469,2.2954 1653 | 3.9666,2.0485 1654 | 3.0673,2.4427 1655 | 3.4375,2.7898 1656 | 3.3208,2.9135 1657 | 3.1341,2.5333 1658 | 3.1346,2.8041 1659 | 3.8059,2.5627 1660 | 3.5248,2.7509 1661 | 3.9443,2.0092 1662 | 3.9883,2.4768 1663 | 3.4099,2.2503 1664 | 3.3712,2.3079 1665 | 3.2269,2.9669 1666 | 3.446,2.2088 1667 | 3.2662,2.5205 1668 | 3.4591,2.2255 1669 | 3.4329,2.5672 1670 | 3.2596,2.9982 1671 | 3.1337,2.1319 1672 | 3.4192,2.9547 1673 | 3.5069,2.1239 1674 | 3.3243,2.1862 1675 | 3.6847,2.6465 1676 | 3.4431,2.1282 1677 | 3.4357,2.0813 1678 | 3.793,2.6592 1679 | 3.8156,2.0274 1680 | 3.7521,2.9852 1681 | 3.7893,2.5393 1682 | 3.5013,2.3738 1683 | 3.5552,2.7067 1684 | 3.6307,2.9474 1685 | 3.098,2.3823 1686 | 3.2457,2.6929 1687 | 3.6157,2.6021 1688 | 3.305,2.7753 1689 | 3.767,2.5918 1690 | 3.2672,2.3762 1691 | 3.0395,2.8506 1692 | 3.2966,2.2257 1693 | 3.5564,2.797 1694 | 3.9691,2.9969 1695 | 3.6891,2.2813 1696 | 3.7179,2.7104 1697 | 3.559,2.6646 1698 | 3.5334,2.4148 1699 | 3.8757,2.4983 1700 | 3.3931,2.9491 1701 | 3.4581,2.9532 1702 | 3.2082,2.7329 1703 | 3.7573,2.3847 1704 | 3.5467,2.0401 1705 | 3.3574,2.5829 1706 | 3.701,2.5647 1707 | 3.1092,2.3552 1708 | 3.0066,2.8802 1709 | 3.5973,2.6245 1710 | 3.6592,2.624 1711 | 3.58,2.2957 1712 | 3.91,2.0747 1713 | 3.636,2.2937 1714 | 3.5256,2.2347 1715 | 3.2596,2.3459 1716 | 3.0512,2.8485 1717 | 3.732,2.1604 1718 | 3.1643,2.1579 1719 | 3.2804,2.5087 1720 | 3.2594,2.6033 1721 | 3.5471,2.1614 1722 | 3.5413,2.6355 1723 | 3.7881,2.8439 1724 | 3.8696,2.7823 1725 | 3.7875,2.2646 1726 | 3.9694,2.3147 1727 | 3.1805,2.1832 1728 | 3.9306,2.4475 1729 | 3.0452,2.3267 1730 | 3.2406,2.2798 1731 | 3.0089,2.9318 1732 | 3.6716,2.3997 1733 | 3.9048,2.3794 1734 | 3.5724,2.5928 1735 | 3.1555,2.0685 1736 | 3.5024,2.2052 1737 | 3.5677,2.7236 1738 | 3.1883,2.5751 1739 | 3.3242,2.2002 1740 | 3.716,2.8435 1741 | 3.5529,2.4237 1742 | 3.1423,2.5448 1743 | 3.3804,2.528 1744 | 3.3966,2.1851 1745 | 3.5767,2.0817 1746 | 3.0194,2.4641 1747 | 3.5776,2.0306 1748 | 3.9322,2.435 1749 | 3.1069,2.5579 1750 | 3.7321,2.6388 1751 | 3.9705,2.0342 1752 | 3.6089,2.7099 1753 | 3.7197,2.1693 1754 | 3.3028,2.5934 1755 | 3.459,2.6081 1756 | 3.048,2.7724 1757 | 3.3854,2.0563 1758 | 3.3617,2.8547 1759 | 3.2876,2.3843 1760 | 3.8167,2.3996 1761 | 3.4505,2.3254 1762 | 3.8066,2.5554 1763 | 3.7902,2.2954 1764 | 3.283,2.3661 1765 | 3.0683,2.349 1766 | 3.0549,2.6302 1767 | 3.6375,2.6644 1768 | 3.4243,2.9921 1769 | 3.9055,2.9444 1770 | 3.4173,2.3503 1771 | 3.1541,2.193 1772 | 3.54,2.9196 1773 | 3.9371,2.2887 1774 | 3.661,2.5509 1775 | 3.3947,2.9193 1776 | 3.259,2.09 1777 | 3.8479,2.2577 1778 | 3.9451,2.427 1779 | 3.377,2.5777 1780 | 3.0673,2.8995 1781 | 3.1816,2.2182 1782 | 3.5757,2.967 1783 | 3.1859,2.434 1784 | 3.2914,2.7848 1785 | 3.4617,2.5252 1786 | 3.347,2.3313 1787 | 3.3182,2.4316 1788 | 3.4599,2.7179 1789 | 3.2359,2.9162 1790 | 3.0278,2.89 1791 | 3.6585,2.1347 1792 | 3.1588,2.1199 1793 | 3.8027,2.8935 1794 | 3.4086,2.6531 1795 | 3.3274,2.0403 1796 | 3.746,2.5047 1797 | 3.7464,2.8945 1798 | 3.174,2.3857 1799 | 3.1175,2.2921 1800 | 3.174,2.234 1801 | 3.6274,2.2009 1802 | 3.8419,2.3803 1803 | 3.5101,2.5948 1804 | 3.1658,2.2684 1805 | 3.7143,2.6224 1806 | 3.907,2.8046 1807 | 3.2185,2.104 1808 | 3.871,2.7292 1809 | 3.2118,2.6486 1810 | 3.8367,2.4747 1811 | 3.8593,2.9329 1812 | 3.5234,2.0964 1813 | 3.4774,2.5991 1814 | 3.8899,2.2336 1815 | 3.0651,2.0323 1816 | 3.5095,2.5799 1817 | 3.6208,2.8422 1818 | 3.7336,2.5569 1819 | 3.23,2.8399 1820 | 3.0219,2.205 1821 | 3.139,2.6213 1822 | 3.7695,2.174 1823 | 3.9698,2.2895 1824 | 3.3868,2.0185 1825 | 3.9934,2.7015 1826 | 3.3264,2.9521 1827 | 3.1372,2.749 1828 | 3.3848,2.7567 1829 | 3.5626,2.5421 1830 | 3.6338,2.2821 1831 | 3.5416,2.2449 1832 | 3.315,2.2863 1833 | 3.1593,2.9631 1834 | 3.1526,2.2307 1835 | 3.137,2.5373 1836 | 3.7098,2.205 1837 | 3.4649,2.434 1838 | 3.1133,2.1422 1839 | 3.7009,2.3756 1840 | 3.18,2.7936 1841 | 3.8037,2.8128 1842 | 3.514,2.9038 1843 | 3.5484,2.5404 1844 | 3.2078,2.8179 1845 | 3.7846,2.7084 1846 | 3.5265,2.0432 1847 | 3.571,2.1459 1848 | 3.422,2.2333 1849 | 3.7212,2.2467 1850 | 3.0731,2.1703 1851 | 3.5949,2.2351 1852 | 3.862,2.2755 1853 | 3.4488,2.9516 1854 | 3.6526,2.3467 1855 | 3.3035,2.2973 1856 | 3.6074,2.4044 1857 | 3.2789,2.3022 1858 | 3.7996,2.7573 1859 | 3.7962,2.3597 1860 | 3.9541,2.1249 1861 | 3.4443,2.6172 1862 | 3.4569,2.3555 1863 | 3.5998,2.3629 1864 | 3.8426,2.0685 1865 | 3.0312,2.8672 1866 | 3.1873,2.4579 1867 | 3.9436,2.0776 1868 | 3.9479,2.9049 1869 | 3.453,2.2817 1870 | 3.8108,2.6139 1871 | 3.9289,2.6619 1872 | 3.6727,2.2 1873 | 3.3723,2.96 1874 | 3.4057,2.6651 1875 | 3.4388,2.5413 1876 | 3.6786,2.869 1877 | 3.4651,2.557 1878 | 3.9533,2.0214 1879 | 3.3547,2.4827 1880 | 3.339,2.808 1881 | 3.8959,2.736 1882 | 3.5454,2.5723 1883 | 3.7493,2.009 1884 | 3.1249,2.7183 1885 | 3.4532,2.4494 1886 | 3.0747,2.6596 1887 | 3.6633,2.7532 1888 | 3.7037,2.8047 1889 | 3.919,2.0292 1890 | 3.6601,2.7798 1891 | 3.6901,2.5674 1892 | 3.8537,2.0761 1893 | 3.4679,2.2516 1894 | 3.4585,2.1335 1895 | 3.8061,2.5645 1896 | 3.8248,2.541 1897 | 3.1904,2.0689 1898 | 3.0257,2.9884 1899 | 3.0568,2.2511 1900 | 3.1429,2.3155 1901 | 3.1714,2.3007 1902 | 3.6258,2.042 1903 | 3.0295,2.5279 1904 | 3.4723,2.256 1905 | 3.6784,2.4087 1906 | 3.1148,2.9475 1907 | 3.2361,2.9193 1908 | 3.2891,2.1212 1909 | 3.1728,2.5919 1910 | 3.3237,2.3597 1911 | 3.8011,2.7193 1912 | 3.2996,2.5236 1913 | 3.7756,2.2608 1914 | 3.5528,2.4931 1915 | 3.5547,2.8558 1916 | 3.7306,2.7244 1917 | 3.7736,2.1991 1918 | 3.9008,2.1573 1919 | 3.1382,2.3705 1920 | 3.7941,2.8623 1921 | 3.1894,2.6848 1922 | 3.029,2.6342 1923 | 3.1274,2.1413 1924 | 3.1337,2.0793 1925 | 3.1283,2.8761 1926 | 3.9353,2.4204 1927 | 3.2732,2.4877 1928 | 3.9427,2.4603 1929 | 3.6382,2.5157 1930 | 3.8725,2.272 1931 | 3.3671,2.2316 1932 | 3.2362,2.8995 1933 | 3.1873,2.9087 1934 | 3.5456,2.6036 1935 | 3.2551,2.3652 1936 | 3.3058,2.5986 1937 | 3.0155,2.6685 1938 | 3.5875,2.8946 1939 | 3.9626,2.0873 1940 | 3.8498,2.539 1941 | 3.0079,2.4284 1942 | 3.634,2.6172 1943 | 3.3593,2.5589 1944 | 3.1141,2.2259 1945 | 3.5408,2.1045 1946 | 3.4164,2.01 1947 | 3.5171,2.0592 1948 | 3.8861,2.3227 1949 | 3.1494,2.7795 1950 | 3.4347,2.3355 1951 | 3.059,2.6196 1952 | 3.381,2.9929 1953 | 3.7224,2.648 1954 | 3.0951,2.5398 1955 | 3.6672,2.2323 1956 | 3.2964,2.7398 1957 | 3.5986,2.889 1958 | 3.1519,2.8598 1959 | 3.4364,2.5971 1960 | 3.0127,2.6548 1961 | 3.229,2.915 1962 | 3.2637,2.4332 1963 | 3.5114,2.2898 1964 | 3.2151,2.6319 1965 | 3.3461,2.2954 1966 | 3.7478,2.622 1967 | 3.4136,2.0475 1968 | 3.0558,2.9946 1969 | 3.39,2.2068 1970 | 3.4745,2.6074 1971 | 3.8253,2.3476 1972 | 3.3036,2.7177 1973 | 3.8218,2.028 1974 | 3.5657,2.0668 1975 | 3.0544,2.9271 1976 | 3.26,2.0878 1977 | 3.5891,2.3324 1978 | 3.4797,2.5262 1979 | 3.1987,2.2466 1980 | 3.239,2.5429 1981 | 3.7802,2.7809 1982 | 3.6173,2.5219 1983 | 3.1441,2.9319 1984 | 3.7161,2.1471 1985 | 3.4015,2.4168 1986 | 3.4624,2.2803 1987 | 3.7073,2.5981 1988 | 3.4012,2.0365 1989 | 3.0144,2.0637 1990 | 3.0746,2.3229 1991 | 3.5911,2.0984 1992 | 3.446,2.17 1993 | 3.9266,2.3712 1994 | 3.0949,2.0398 1995 | 3.3754,2.7092 1996 | 3.546,2.6413 1997 | 3.1117,2.1741 1998 | 3.9045,2.0622 1999 | 3.6333,2.4067 2000 | 3.9054,2.4631 2001 | -------------------------------------------------------------------------------- /data/wordcount.data: -------------------------------------------------------------------------------- 1 | In computer science, functional programming is a programming paradigm — a style 2 | of building the structure and elements of computer programs — that treats 3 | computation as the evaluation of mathematical functions and avoids changing 4 | state and mutable data. It is a declarative programming paradigm, which means 5 | programming is done with expressions or declarations[2] instead of statements. 6 | In functional code, the output value of a function depends only on the arguments 7 | that are input to the function, so calling a function f twice with the same 8 | value for an argument x will produce the same result f(x) each time. Eliminating 9 | side effects, i.e. changes in state that do not depend on the function inputs, 10 | can make it much easier to understand and predict the behaviour of a program, 11 | which is one of the key motivations for the development of functional 12 | programming. 13 | -------------------------------------------------------------------------------- /jbuild-ignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/owlbarn/actor/878f4588f4ee235994bbc4e77648858bb562bdd2/jbuild-ignore -------------------------------------------------------------------------------- /jbuild-workspace: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/owlbarn/actor/878f4588f4ee235994bbc4e77648858bb562bdd2/jbuild-workspace -------------------------------------------------------------------------------- /lib/actor.ml: -------------------------------------------------------------------------------- 1 | (* 2 | * Actor - Parallel & Distributed Engine of Owl System 3 | * Copyright (c) 2016-2018 Liang Wang 4 | *) 5 | 6 | (* module aliases *) 7 | 8 | module Mapre = Actor_mapre 9 | 10 | module Param = Actor_param 11 | 12 | module Peer = Actor_peer 13 | -------------------------------------------------------------------------------- /lib/actor_barrier.ml: -------------------------------------------------------------------------------- 1 | (* 2 | * Actor - Parallel & Distributed Engine of Owl System 3 | * Copyright (c) 2016-2018 Liang Wang 4 | *) 5 | 6 | (* Barrier module provides flexible synchronisation barrier controls. *) 7 | 8 | open Actor_types 9 | 10 | 11 | (* Mapre barrier: Bulk synchronous parallel *) 12 | let mapre_bsp bar _context = 13 | let h = Hashtbl.create 1024 in 14 | (* first check the buffer for those arrive early *) 15 | List.iter (fun (i,m) -> 16 | if not (Hashtbl.mem h i) then Hashtbl.(add h i m; remove !_context.msbuf bar) 17 | ) (Hashtbl.find_all !_context.msbuf bar); 18 | (* then wait for the rest of the messages *) 19 | while (Hashtbl.length h) < (StrMap.cardinal !_context.workers) do 20 | let i, m = Actor_utils.recv !_context.myself_sock in 21 | if bar = m.bar && not (Hashtbl.mem h i) then Hashtbl.add h i m; 22 | done; 23 | Hashtbl.fold (fun _k v l -> v :: l) h [] 24 | 25 | 26 | (* Mapre barrier: Delay bounded parallel *) 27 | let mapre_dbp bar _context = 28 | let h = Hashtbl.create 1024 in 29 | (* first check the buffer for those arrive early *) 30 | List.iter (fun (i,m) -> 31 | if not (Hashtbl.mem h i) then Hashtbl.(add h i m; remove !_context.msbuf bar) 32 | ) (Hashtbl.find_all !_context.msbuf bar); 33 | (* then wait for the rest of the messages *) 34 | let budget = 0.001 in 35 | let t0 = Unix.gettimeofday () in 36 | (try while (Hashtbl.length h) < (StrMap.cardinal !_context.workers) do 37 | let i, m = Actor_utils.recv !_context.myself_sock in 38 | if bar = m.bar && not (Hashtbl.mem h i) then Hashtbl.add h i m; 39 | if budget < (Unix.gettimeofday () -. t0) then failwith "timeout" 40 | done 41 | with _exn -> Owl_log.info "%s" "timeout +++"); 42 | Hashtbl.fold (fun _k v l -> v :: l) h [] 43 | 44 | 45 | (* Param barrier: Bulk synchronous parallel *) 46 | let param_bsp _context = 47 | let num_finish = List.length (Hashtbl.find_all !_context.step_worker !_context.step) in 48 | let num_worker = StrMap.cardinal !_context.workers in 49 | match num_finish = num_worker with 50 | | true -> !_context.step + 1, (StrMap.keys !_context.workers) 51 | | false -> !_context.step, [] 52 | 53 | 54 | (* Param barrier: Stale synchronous parallel *) 55 | let param_ssp _context = 56 | let num_finish = List.length (Hashtbl.find_all !_context.step_worker !_context.step) in 57 | let num_worker = StrMap.cardinal !_context.workers in 58 | let t = match num_finish = num_worker with 59 | | true -> !_context.step + 1 60 | | false -> !_context.step 61 | in 62 | let l = Hashtbl.fold (fun w t' l -> 63 | let busy = Hashtbl.find !_context.worker_busy w in 64 | match (busy = 0) && ((t' - t) < !_context.stale) with 65 | | true -> l @ [ w ] 66 | | false -> l 67 | ) !_context.worker_step [] 68 | in (t, l) 69 | 70 | 71 | (* Param barrier: Asynchronous parallel *) 72 | let param_asp _context = !_context.stale <- max_int; param_ssp _context 73 | 74 | 75 | (* P2P barrier : Bulk synchronous parallel 76 | this one waits for the slowest one to catch up. *) 77 | let p2p_bsp _context = 78 | match !_context.block with 79 | | true -> ( 80 | List.for_all (fun k -> 81 | let t = Hashtbl.find !_context.spbuf k in 82 | t >= !_context.step 83 | ) (StrMap.keys !_context.workers) 84 | ) 85 | | false -> false 86 | 87 | 88 | (* P2P barrier: Asynchronous parallel *) 89 | let p2p_asp _context = true 90 | 91 | 92 | (* P2P barrier: Asynchronous parallel but aligned with local client *) 93 | let p2p_asp_local _context = !_context.block 94 | -------------------------------------------------------------------------------- /lib/actor_config.ml: -------------------------------------------------------------------------------- 1 | (* 2 | * Actor - Parallel & Distributed Engine of Owl System 3 | * Copyright (c) 2016-2018 Liang Wang 4 | *) 5 | 6 | (* Config: contains the static configurations of the framework. *) 7 | 8 | 9 | (** Manager's address, all workders connect to this address *) 10 | let manager_addr = "tcp://127.0.0.1:5555" 11 | 12 | 13 | (** Manager's identifier *) 14 | let manager_id = "manager_0" 15 | 16 | 17 | (** File system path, Irmin or HDFS *) 18 | let dfs_path = "storage.data" 19 | 20 | 21 | (** Log configs: path, level, color, etc. *) 22 | let update_config level logdir fname = 23 | Owl_log.set_level level; 24 | match fname with 25 | | "" -> (Owl_log.set_color true; Owl_log.set_output stderr) 26 | | _ -> (Owl_log.set_color false; open_out (logdir^fname) |> Owl_log.set_output) 27 | 28 | let logdir = "log/" 29 | let _ = update_config Owl_log.DEBUG logdir "" 30 | 31 | 32 | (** Max queue length of ZMQ send and receive *) 33 | let high_warter_mark = 10_000 34 | 35 | 36 | (** WebHDFS base addr and port *) 37 | let webhdfs_addr = "192.168.99.100:50070" 38 | -------------------------------------------------------------------------------- /lib/actor_dag.ml: -------------------------------------------------------------------------------- 1 | (* 2 | * Actor - Parallel & Distributed Engine of Owl System 3 | * Copyright (c) 2016-2018 Liang Wang 4 | *) 5 | 6 | open Actor_types 7 | 8 | 9 | type vlabel = { c : color; f : string } 10 | 11 | 12 | module Digraph = struct 13 | module V' = struct 14 | type t = string 15 | let hash = Hashtbl.hash 16 | let equal = (=) 17 | let compare = Pervasives.compare 18 | end 19 | module E' = struct 20 | type t = int 21 | let compare = Pervasives.compare 22 | let default = 0 23 | end 24 | include Graph.Imperative.Digraph.ConcreteLabeled (V') (E') 25 | end 26 | 27 | 28 | module TopoOrd = Graph.Topological.Make_stable (Digraph) 29 | 30 | 31 | let _graph = ref (Digraph.create ()) 32 | 33 | 34 | let _vlabel = Hashtbl.create 1048576 35 | 36 | 37 | let get_vlabel_f x = (Hashtbl.find _vlabel x).f 38 | 39 | 40 | let add_edge f u v c = 41 | if (Hashtbl.mem _vlabel u) = false then 42 | Hashtbl.add _vlabel u { c = Green; f = "" }; 43 | Hashtbl.add _vlabel v { c = c; f = f }; 44 | Digraph.add_edge !_graph u v 45 | 46 | 47 | let stages_eager () = 48 | let r, s = ref [], ref [] in 49 | let _ = TopoOrd.iter (fun v -> 50 | match (Hashtbl.find _vlabel v).c with 51 | | Blue -> s := !s @ [v]; r := !r @ [!s]; s := [] 52 | | Red -> s := !s @ [v] 53 | | Green -> () 54 | ) !_graph in 55 | if List.length !s = 0 then !r else !r @ [!s] 56 | 57 | 58 | let rec _stages_lazy v = 59 | let l = ref [v] in 60 | List.iter (fun u -> 61 | match (Hashtbl.find _vlabel u).c with 62 | | Blue | Red -> l := (_stages_lazy u) @ !l 63 | | Green -> () 64 | ) (Digraph.pred !_graph v); !l 65 | 66 | 67 | let stages_lazy v = 68 | let r, s = ref [], ref [] in 69 | let _ = List.iter (fun v -> 70 | match (Hashtbl.find _vlabel v).c with 71 | | Blue -> s := !s @ [v]; r := !r @ [!s]; s := [] 72 | | Red -> s := !s @ [v] 73 | | Green -> () 74 | ) (_stages_lazy v) in 75 | if List.length !s = 0 then !r else !r @ [!s] 76 | 77 | 78 | let mark_stage_done s = 79 | List.iter (fun k -> 80 | let v = Hashtbl.find _vlabel k in 81 | Hashtbl.add _vlabel k { c = Green; f = v.f } 82 | ) s 83 | 84 | 85 | (* FIXME: the following functions are for debugging *) 86 | 87 | let print_vertex v = 88 | let x = Hashtbl.find _vlabel v in 89 | match x.c with 90 | | Red -> Printf.printf "(%s, Red); " v 91 | | Green -> Printf.printf "(%s, Green); " v 92 | | Blue -> Printf.printf "(%s, Blue); " v 93 | 94 | 95 | let print_stages x = 96 | print_endline ""; 97 | List.iter (fun l -> 98 | print_string "stage: "; 99 | List.iter (fun v -> 100 | print_vertex v 101 | ) l; print_endline "" 102 | ) x 103 | 104 | 105 | let print_tasks () = TopoOrd.iter (fun v -> print_vertex v) !_graph 106 | 107 | 108 | let test () = 109 | add_edge "" "1" "2" Red; 110 | add_edge "" "1" "4" Blue; 111 | add_edge "" "1" "6" Red; 112 | add_edge "" "2" "3" Red; 113 | add_edge "" "4" "3" Red; 114 | add_edge "" "3" "5" Blue; 115 | add_edge "" "4" "7" Red; 116 | add_edge "" "6" "8" Blue; 117 | add_edge "" "7" "8" Blue; 118 | TopoOrd.iter (fun v -> print_vertex v) !_graph; 119 | print_stages (stages_lazy "5"); 120 | -------------------------------------------------------------------------------- /lib/actor_mapre.ml: -------------------------------------------------------------------------------- 1 | (* 2 | * Actor - Parallel & Distributed Engine of Owl System 3 | * Copyright (c) 2016-2018 Liang Wang 4 | *) 5 | 6 | (* Data Parallel: Map-Reduce module *) 7 | 8 | open Actor_types 9 | 10 | 11 | let init jid url = 12 | let _ztx = ZMQ.Context.create () in 13 | let _addr, _router = Actor_utils.bind_available_addr _ztx in 14 | let req = ZMQ.Socket.create _ztx ZMQ.Socket.req in 15 | ZMQ.Socket.connect req url; 16 | Actor_utils.send req Job_Reg [|_addr; jid|]; 17 | (* create and initialise part of the context *) 18 | let _context = Actor_utils.empty_mapre_context () in 19 | _context.job_id <- jid; 20 | _context.myself_addr <- _addr; 21 | _context.myself_sock <- _router; 22 | _context.ztx <- _ztx; 23 | (* depends on the role, start server or client *) 24 | let m = of_msg (ZMQ.Socket.recv req) in 25 | let _ = match m.typ with 26 | | Job_Master -> Actor_mapreserver.init m _context 27 | | Job_Worker -> Actor_mapreclient.init m _context 28 | | _ -> Owl_log.info "%s" "unknown command" 29 | in 30 | ZMQ.Socket.close req 31 | 32 | 33 | (* interface to mapreserver functions *) 34 | 35 | let map = Actor_mapreserver.map 36 | 37 | 38 | let map_partition = Actor_mapreserver.map_partition 39 | 40 | 41 | let flatmap = Actor_mapreserver.flatmap 42 | 43 | 44 | let reduce = Actor_mapreserver.reduce 45 | 46 | 47 | let reduce_by_key = Actor_mapreserver.reduce_by_key 48 | 49 | 50 | let fold = Actor_mapreserver.fold 51 | 52 | 53 | let filter = Actor_mapreserver.filter 54 | 55 | 56 | let flatten = Actor_mapreserver.flatten 57 | 58 | 59 | let shuffle = Actor_mapreserver.shuffle 60 | 61 | 62 | let union = Actor_mapreserver.union 63 | 64 | 65 | let join = Actor_mapreserver.join 66 | 67 | 68 | let broadcast = Actor_mapreserver.broadcast 69 | 70 | 71 | let get_value = Actor_mapreserver.get_value 72 | 73 | 74 | let count = Actor_mapreserver.count 75 | 76 | 77 | let collect = Actor_mapreserver.collect 78 | 79 | 80 | let terminate = Actor_mapreserver.terminate 81 | 82 | 83 | let apply = Actor_mapreserver.apply 84 | 85 | 86 | let load = Actor_mapreserver.load 87 | 88 | 89 | let save = Actor_mapreserver.save 90 | 91 | 92 | (* experimental functions *) 93 | 94 | let workers = Actor_mapreserver.workers 95 | 96 | 97 | let myself () = 98 | match Actor_mapreserver.(!_context.job_id) = "" with 99 | | true -> Actor_mapreclient.(!_context.myself_addr) 100 | | false -> Actor_mapreserver.(!_context.myself_addr) 101 | -------------------------------------------------------------------------------- /lib/actor_mapre.mli: -------------------------------------------------------------------------------- 1 | (* 2 | * Actor - Parallel & Distributed Engine of Owl System 3 | * Copyright (c) 2016-2018 Liang Wang 4 | *) 5 | 6 | (* Data Parallel: Map-Reduce module *) 7 | 8 | val init : string -> string -> unit 9 | 10 | val map : ('a -> 'b) -> string -> string 11 | 12 | val map_partition : ('a list -> 'b list) -> string -> string 13 | 14 | val flatmap : ('a -> 'b list) -> string -> string 15 | 16 | val reduce : ('a -> 'a -> 'a) -> string -> 'a option 17 | 18 | val reduce_by_key : ('a -> 'a -> 'a) -> string -> string 19 | 20 | val fold : ('a -> 'b -> 'a) -> 'a -> string -> 'a 21 | 22 | val filter : ('a -> bool) -> string -> string 23 | 24 | val flatten : string -> string 25 | 26 | val shuffle : string -> string 27 | 28 | val union : string -> string -> string 29 | 30 | val join : string -> string -> string 31 | 32 | val broadcast : 'a -> string 33 | 34 | val get_value : string -> 'a 35 | 36 | val count : string -> int 37 | 38 | val collect : string -> 'a list 39 | 40 | val terminate : unit -> unit 41 | 42 | val apply : ('a list -> 'b list) -> string list -> string list -> string list 43 | 44 | val load : string -> string 45 | 46 | val save : string -> string -> int 47 | 48 | 49 | (* experimental functions *) 50 | 51 | val workers : unit -> string list 52 | 53 | val myself : unit -> string 54 | 55 | 56 | (** TODO: sample function *) 57 | -------------------------------------------------------------------------------- /lib/actor_mapreclient.ml: -------------------------------------------------------------------------------- 1 | (* 2 | * Actor - Parallel & Distributed Engine of Owl System 3 | * Copyright (c) 2016-2018 Liang Wang 4 | *) 5 | 6 | (** Data Parallel: Map-Reduce client module *) 7 | 8 | open Actor_types 9 | 10 | 11 | (* the global context: master, worker, etc. *) 12 | let _context = ref (Actor_utils.empty_mapre_context ()) 13 | 14 | 15 | let barrier bar = Actor_barrier.mapre_bsp bar _context 16 | 17 | 18 | let shuffle bar x z = 19 | List.mapi (fun i k -> 20 | let v = Actor_utils.choose_load x (List.length z) i in 21 | let s = if StrMap.mem k !_context.workers then 22 | StrMap.find k !_context.workers 23 | else ( 24 | let s = ZMQ.Socket.(create !_context.ztx dealer) in 25 | let _ = ZMQ.Socket.(set_identity s !_context.myself_addr; connect s k) in 26 | let _ = !_context.workers <- StrMap.add k s !_context.workers in 27 | let _ = ZMQ.Socket.set_send_high_water_mark s Actor_config.high_warter_mark in 28 | s ) in 29 | Actor_utils.send ~bar s OK [|Marshal.to_string v []|] 30 | ) z 31 | 32 | 33 | let process_pipeline s = 34 | Array.iter (fun s -> 35 | let m = of_msg s in 36 | match m.typ with 37 | | MapTask -> ( 38 | Owl_log.info "%s" ("map @ " ^ !_context.myself_addr); 39 | let f : 'a -> 'b = Marshal.from_string m.par.(0) 0 in 40 | List.map f (Actor_memory.find m.par.(1)) |> Actor_memory.add m.par.(2) 41 | ) 42 | | MapPartTask -> ( 43 | Owl_log.info "%s" ("map_partition @ " ^ !_context.myself_addr); 44 | let f : 'a list -> 'b list = Marshal.from_string m.par.(0) 0 in 45 | f (Actor_memory.find m.par.(1)) |> Actor_memory.add m.par.(2) 46 | ) 47 | | FilterTask -> ( 48 | Owl_log.info "%s" ("filter @ " ^ !_context.myself_addr); 49 | let f : 'a -> bool = Marshal.from_string m.par.(0) 0 in 50 | List.filter f (Actor_memory.find m.par.(1)) |> Actor_memory.add m.par.(2) 51 | ) 52 | | FlattenTask -> ( 53 | Owl_log.info "%s" ("flatten @ " ^ !_context.myself_addr); 54 | List.flatten (Actor_memory.find m.par.(0)) |> Actor_memory.add m.par.(1) 55 | ) 56 | | UnionTask -> ( 57 | Owl_log.info "%s" ("union @ " ^ !_context.myself_addr); 58 | (Actor_memory.find m.par.(0)) @ (Actor_memory.find m.par.(1)) 59 | |> Actor_memory.add m.par.(2) 60 | ) 61 | | ReduceByKeyTask -> ( 62 | Owl_log.info "%s" ("reduce_by_key @ " ^ !_context.myself_addr); 63 | let f : 'a -> 'a -> 'a = Marshal.from_string m.par.(0) 0 in 64 | Actor_memory.find m.par.(1) |> Actor_utils.group_by_key |> List.map (fun (k,l) -> 65 | match l with 66 | | hd :: tl -> (k, List.fold_left f hd tl) 67 | | [] -> failwith "error in reduce" 68 | ) 69 | |> Actor_memory.add m.par.(2) 70 | ) 71 | | JoinTask -> ( 72 | Owl_log.info "%s" ("join @ " ^ !_context.myself_addr); 73 | (Actor_memory.find m.par.(0)) @ (Actor_memory.find m.par.(1)) 74 | |> Actor_utils.group_by_key |> Actor_memory.add m.par.(2) 75 | ) 76 | | ShuffleTask -> ( 77 | Owl_log.info "%s" ("shuffle @ " ^ !_context.myself_addr); 78 | let x = Actor_memory.find m.par.(0) |> Actor_utils.group_by_key in 79 | let z = Marshal.from_string m.par.(2) 0 in 80 | let bar = Marshal.from_string m.par.(3) 0 in 81 | let _ = shuffle bar x z in 82 | barrier bar 83 | |> List.map (fun m -> Marshal.from_string m.par.(0) 0 |> Actor_utils.flatten_kvg) 84 | |> List.flatten |> Actor_memory.add m.par.(1); 85 | ) 86 | | _ -> Owl_log.info "%s" "unknown task types" 87 | ) s 88 | 89 | 90 | let service_loop () = 91 | Owl_log.debug "mapre worker @ %s" !_context.myself_addr; 92 | (* set up local loop of a job worker *) 93 | try while true do 94 | let i, m = Actor_utils.recv !_context.myself_sock in 95 | let bar = m.bar in 96 | match m.typ with 97 | | Count -> ( 98 | Owl_log.info "%s" ("count @ " ^ !_context.myself_addr); 99 | let y = List.length (Actor_memory.find m.par.(0)) in 100 | Actor_utils.send ~bar !_context.master_sock OK [|Marshal.to_string y []|] 101 | ) 102 | | Collect -> ( 103 | Owl_log.info "%s" ("collect @ " ^ !_context.myself_addr); 104 | let y = Actor_memory.find m.par.(0) in 105 | Actor_utils.send ~bar !_context.master_sock OK [|Marshal.to_string y []|] 106 | ) 107 | | Broadcast -> ( 108 | Owl_log.info "%s" ("broadcast @ " ^ !_context.myself_addr); 109 | Actor_memory.add m.par.(1) (Marshal.from_string m.par.(0) 0); 110 | Actor_utils.send ~bar !_context.master_sock OK [||] 111 | ) 112 | | Reduce -> ( 113 | Owl_log.info "%s" ("reduce @ " ^ !_context.myself_addr); 114 | let f : 'a -> 'a -> 'a = Marshal.from_string m.par.(0) 0 in 115 | let y = 116 | match Actor_memory.find m.par.(1) with 117 | | hd :: tl -> Some (List.fold_left f hd tl) 118 | | [] -> None 119 | in 120 | Actor_utils.send ~bar !_context.master_sock OK [|Marshal.to_string y []|]; 121 | ) 122 | | Fold -> ( 123 | Owl_log.info "%s" ("fold @ " ^ !_context.myself_addr); 124 | let f : 'a -> 'b -> 'a = Marshal.from_string m.par.(0) 0 in 125 | let y = 126 | match Actor_memory.find m.par.(1) with 127 | | hd :: tl -> Some (List.fold_left f hd tl) 128 | | [] -> None 129 | in 130 | Actor_utils.send ~bar !_context.master_sock OK [|Marshal.to_string y []|]; 131 | ) 132 | | Pipeline -> ( 133 | Owl_log.info "%s" ("pipelined @ " ^ !_context.myself_addr); 134 | process_pipeline m.par; 135 | Actor_utils.send ~bar !_context.master_sock OK [||] 136 | ) 137 | | Terminate -> ( 138 | Owl_log.info "%s" ("terminate @ " ^ !_context.myself_addr); 139 | Actor_utils.send ~bar !_context.master_sock OK [||]; 140 | Unix.sleep 1; (* FIXME: sleep ... *) 141 | failwith ("#" ^ !_context.job_id ^ " terminated") 142 | ) 143 | | Load -> ( 144 | Owl_log.info "%s" ("load " ^ m.par.(0) ^ " @ " ^ !_context.myself_addr); 145 | let path = Str.(split (regexp "://")) m.par.(0) in 146 | let b = 147 | match (List.nth path 0) with 148 | | "unix" -> Actor_storage.unix_load (List.nth path 1) 149 | | _ -> failwith "Load: unknown system!" 150 | in 151 | Actor_memory.add m.par.(1) [ b ]; 152 | Actor_utils.send ~bar !_context.master_sock OK [||] 153 | ) 154 | | Save -> ( 155 | Owl_log.info "%s" ("save " ^ m.par.(0) ^ " @ " ^ !_context.myself_addr); 156 | let path = Str.(split (regexp "://")) m.par.(0) in 157 | let c = 158 | match (List.nth path 0) with 159 | | "unix" -> Actor_storage.unix_save (List.nth path 1) (Actor_memory.find m.par.(1)) 160 | | _ -> failwith "Save: unknown system!" 161 | in 162 | Actor_utils.send ~bar !_context.master_sock OK [|Marshal.to_string c []|] 163 | ) 164 | | _ -> ( 165 | Owl_log.info "%s" ("Buffering " ^ !_context.myself_addr ^ " <- " ^ i ^ " m.bar : " ^ string_of_int (m.bar)); 166 | Hashtbl.add !_context.msbuf m.bar (i,m) 167 | ) 168 | done with Failure e -> ( 169 | Owl_log.warn "%s" e; 170 | ZMQ.Socket.(close !_context.master_sock; close !_context.myself_sock); 171 | Pervasives.exit 0 ) 172 | 173 | 174 | let init m context = 175 | _context := context; 176 | !_context.master_addr <- m.par.(0); 177 | (* connect to job master *) 178 | let master = ZMQ.Socket.create !_context.ztx ZMQ.Socket.dealer in 179 | ZMQ.Socket.set_send_high_water_mark master Actor_config.high_warter_mark; 180 | ZMQ.Socket.set_identity master !_context.myself_addr; 181 | ZMQ.Socket.connect master !_context.master_addr; 182 | Actor_utils.send master OK [|!_context.myself_addr|]; 183 | !_context.master_sock <- master; 184 | (* enter into worker service loop *) 185 | service_loop () 186 | -------------------------------------------------------------------------------- /lib/actor_mapreserver.ml: -------------------------------------------------------------------------------- 1 | (* 2 | * Actor - Parallel & Distributed Engine of Owl System 3 | * Copyright (c) 2016-2018 Liang Wang 4 | *) 5 | 6 | (* Data Parallel: Map-Reduce server module *) 7 | 8 | open Actor_types 9 | 10 | 11 | (* the global context: master, worker, etc. *) 12 | let _context = ref (Actor_utils.empty_mapre_context ()) 13 | 14 | 15 | let barrier bar = Actor_barrier.mapre_bsp bar _context 16 | 17 | 18 | let _broadcast_all t s = 19 | let bar = Random.int 536870912 in 20 | StrMap.iter (fun _k v -> Actor_utils.send ~bar v t s) !_context.workers; 21 | bar 22 | 23 | 24 | let run_job_eager () = 25 | List.iter (fun s -> 26 | let s' = List.map (fun x -> Actor_dag.get_vlabel_f x) s in 27 | let bar = _broadcast_all Pipeline (Array.of_list s') in 28 | let _ = barrier bar in 29 | Actor_dag.mark_stage_done s; 30 | ) (Actor_dag.stages_eager ()) 31 | 32 | 33 | let run_job_lazy x = 34 | List.iter (fun s -> 35 | let s' = List.map (fun x -> Actor_dag.get_vlabel_f x) s in 36 | let bar = _broadcast_all Pipeline (Array.of_list s') in 37 | let _ = barrier bar in 38 | Actor_dag.mark_stage_done s; 39 | ) (Actor_dag.stages_lazy x) 40 | 41 | 42 | let collect x = 43 | Owl_log.info "%s" ("collect " ^ x ^ "\n"); 44 | run_job_lazy x; 45 | let bar = _broadcast_all Collect [|x|] in 46 | barrier bar 47 | |> List.map (fun m -> Marshal.from_string m.par.(0) 0) 48 | 49 | 50 | let count x = 51 | Owl_log.info "%s" ("count " ^ x ^ "\n"); 52 | run_job_lazy x; 53 | let bar = _broadcast_all Count [|x|] in 54 | barrier bar 55 | |> List.map (fun m -> Marshal.from_string m.par.(0) 0) 56 | |> List.fold_left (+) 0 57 | 58 | 59 | let fold f a x = 60 | Owl_log.info "%s" ("fold " ^ x ^ "\n"); 61 | run_job_lazy x; 62 | let g = Marshal.to_string f [ Marshal.Closures ] in 63 | let bar = _broadcast_all Fold [|g; x|] in 64 | barrier bar 65 | |> List.map (fun m -> Marshal.from_string m.par.(0) 0) 66 | |> List.filter (function Some _x -> true | None -> false) 67 | |> List.map (function Some x -> x | None -> failwith "") 68 | |> List.fold_left f a 69 | 70 | 71 | let reduce f x = 72 | Owl_log.info "%s" ("reduce " ^ x ^ "\n"); 73 | run_job_lazy x; 74 | let g = Marshal.to_string f [ Marshal.Closures ] in 75 | let bar = _broadcast_all Reduce [|g; x|] in 76 | let y = barrier bar 77 | |> List.map (fun m -> Marshal.from_string m.par.(0) 0) 78 | |> List.filter (function Some _x -> true | None -> false) 79 | |> List.map (function Some x -> x | None -> failwith "") in 80 | match y with 81 | | hd :: tl -> Some (List.fold_left f hd tl) 82 | | [] -> None 83 | 84 | 85 | let terminate () = 86 | Owl_log.info "%s" ("terminate #" ^ !_context.job_id ^ "\n"); 87 | let bar = _broadcast_all Terminate [||] in 88 | let _ = barrier bar in () 89 | 90 | 91 | let broadcast x = 92 | Owl_log.info "%s" ("broadcast -> " ^ string_of_int (StrMap.cardinal !_context.workers) ^ " workers\n"); 93 | let y = Actor_memory.rand_id () in 94 | let bar = _broadcast_all Broadcast [|Marshal.to_string x []; y|] in 95 | let _ = barrier bar in y 96 | 97 | 98 | let get_value x = Actor_memory.find x 99 | 100 | 101 | let map f x = 102 | let y = Actor_memory.rand_id () in 103 | Owl_log.info "%s" ("map " ^ x ^ " -> " ^ y ^ "\n"); 104 | let g = Marshal.to_string f [ Marshal.Closures ] in 105 | Actor_dag.add_edge (to_msg 0 MapTask [|g; x; y|]) x y Red; y 106 | 107 | 108 | let map_partition f x = 109 | let y = Actor_memory.rand_id () in 110 | Owl_log.info "%s" ("map_partition " ^ x ^ " -> " ^ y ^ "\n"); 111 | let g = Marshal.to_string f [ Marshal.Closures ] in 112 | Actor_dag.add_edge (to_msg 0 MapPartTask [|g; x; y|]) x y Red; y 113 | 114 | 115 | let filter f x = 116 | let y = Actor_memory.rand_id () in 117 | Owl_log.info "%s" ("filter " ^ x ^ " -> " ^ y ^ "\n"); 118 | let g = Marshal.to_string f [ Marshal.Closures ] in 119 | Actor_dag.add_edge (to_msg 0 FilterTask [|g; x; y|]) x y Red; y 120 | 121 | 122 | let flatten x = 123 | let y = Actor_memory.rand_id () in 124 | Owl_log.info "%s" ("flatten " ^ x ^ " -> " ^ y ^ "\n"); 125 | Actor_dag.add_edge (to_msg 0 FlattenTask [|x; y|]) x y Red; y 126 | 127 | 128 | let flatmap f x = flatten (map f x) 129 | 130 | 131 | let union x y = 132 | let z = Actor_memory.rand_id () in 133 | Owl_log.info "%s" ("union " ^ x ^ " & " ^ y ^ " -> " ^ z ^ "\n"); 134 | Actor_dag.add_edge (to_msg 0 UnionTask [|x; y; z|]) x z Red; 135 | Actor_dag.add_edge (to_msg 0 UnionTask [|x; y; z|]) y z Red; z 136 | 137 | 138 | let shuffle x = 139 | let y = Actor_memory.rand_id () in 140 | Owl_log.info "%s" ("shuffle " ^ x ^ " -> " ^ y ^ "\n"); 141 | let z = Marshal.to_string (StrMap.keys !_context.workers) [] in 142 | let b = Marshal.to_string (Random.int 536870912) [] in 143 | Actor_dag.add_edge (to_msg 0 ShuffleTask [|x; y; z; b|]) x y Blue; y 144 | 145 | 146 | let reduce_by_key f x = 147 | (* TODO: without local combiner ... keep or not? *) 148 | let x = shuffle x in 149 | let y = Actor_memory.rand_id () in 150 | Owl_log.info "%s" ("reduce_by_key " ^ x ^ " -> " ^ y ^ "\n"); 151 | let g = Marshal.to_string f [ Marshal.Closures ] in 152 | Actor_dag.add_edge (to_msg 0 ReduceByKeyTask [|g; x; y|]) x y Red; y 153 | 154 | 155 | let ___reduce_by_key f x = 156 | (* TODO: with local combiner ... keep or not? *) 157 | let g = Marshal.to_string f [ Marshal.Closures ] in 158 | let y = Actor_memory.rand_id () in 159 | Actor_dag.add_edge (to_msg 0 ReduceByKeyTask [|g; x; y|]) x y Red; 160 | let x = shuffle y in 161 | let y = Actor_memory.rand_id () in 162 | Owl_log.info "%s" ("reduce_by_key " ^ x ^ " -> " ^ y ^ "\n"); 163 | Actor_dag.add_edge (to_msg 0 ReduceByKeyTask [|g; x; y|]) x y Red; y 164 | 165 | 166 | let join x y = 167 | let z = Actor_memory.rand_id () in 168 | Owl_log.info "%s" ("join " ^ x ^ " & " ^ y ^ " -> " ^ z ^ "\n"); 169 | let x, y = shuffle x, shuffle y in 170 | Actor_dag.add_edge (to_msg 0 JoinTask [|x; y; z|]) x z Red; 171 | Actor_dag.add_edge (to_msg 0 JoinTask [|x; y; z|]) y z Red; z 172 | 173 | 174 | let apply f i o = 175 | Owl_log.info "%s" ("apply f ... " ^ "\n"); 176 | let g = Marshal.to_string f [ Marshal.Closures ] in 177 | let o = List.map (fun _ -> Actor_memory.rand_id ()) o in 178 | let x = Marshal.to_string i [ ] in 179 | let y = Marshal.to_string o [ ] in 180 | let z = Actor_memory.rand_id () in 181 | List.iter (fun m -> Actor_dag.add_edge (to_msg 0 ApplyTask [|g; x; z; y|]) m z Red) i; 182 | List.iter (fun n -> Actor_dag.add_edge (to_msg 0 NopTask [|z; y|]) z n Red) o; o 183 | 184 | 185 | let load x = 186 | Owl_log.info "%s" ("load " ^ x ^ "\n"); 187 | let y = Actor_memory.rand_id () in 188 | let bar = _broadcast_all Load [|x; y|] in 189 | let _ = barrier bar in y 190 | 191 | 192 | let save x y = 193 | Owl_log.info "%s" ("save " ^ x ^ "\n"); 194 | let bar = _broadcast_all Save [|x; y|] in 195 | barrier bar 196 | |> List.map (fun m -> Marshal.from_string m.par.(0) 0) 197 | |> List.fold_left (+) 0 198 | 199 | 200 | let init m context = 201 | _context := context; 202 | (* contact allocated actors to assign jobs *) 203 | let addrs = Marshal.from_string m.par.(0) 0 in 204 | List.map (fun x -> 205 | let req = ZMQ.Socket.create !_context.ztx ZMQ.Socket.req in 206 | ZMQ.Socket.connect req x; 207 | let app = Filename.basename Sys.argv.(0) in 208 | let arg = Marshal.to_string Sys.argv [] in 209 | Actor_utils.send req Job_Create [|!_context.myself_addr; app; arg|]; req 210 | ) addrs |> List.iter ZMQ.Socket.close; 211 | (* wait until all the allocated actors register *) 212 | while (StrMap.cardinal !_context.workers) < (List.length addrs) do 213 | let _i, m = Actor_utils.recv !_context.myself_sock in 214 | let s = ZMQ.Socket.create !_context.ztx ZMQ.Socket.dealer in 215 | ZMQ.Socket.connect s m.par.(0); 216 | !_context.workers <- (StrMap.add m.par.(0) s !_context.workers); 217 | done 218 | 219 | 220 | (* experimental functions *) 221 | 222 | let workers () = StrMap.keys !_context.workers 223 | 224 | 225 | (* ends here *) 226 | -------------------------------------------------------------------------------- /lib/actor_memory.ml: -------------------------------------------------------------------------------- 1 | (* 2 | * Actor - Parallel & Distributed Engine of Owl System 3 | * Copyright (c) 2016-2018 Liang Wang 4 | *) 5 | 6 | (* Dfs is a distributed file system for actors *) 7 | 8 | let _data : (string, Obj.t) Hashtbl. t = Hashtbl.create 10_000_000 9 | 10 | 11 | let rand_id () = string_of_int (Random.int 536_870_912) 12 | 13 | 14 | let mem id = Hashtbl.mem _data id 15 | 16 | 17 | let add id d = Hashtbl.add _data id (Obj.repr d) 18 | 19 | 20 | let remove id = Hashtbl.remove _data id 21 | 22 | 23 | let find id = match id with 24 | | "" -> Obj.obj (Obj.repr [ None ]) 25 | | _ -> Obj.obj (Hashtbl.find _data id) 26 | 27 | 28 | let size id = Obj.size (find id) 29 | -------------------------------------------------------------------------------- /lib/actor_param.ml: -------------------------------------------------------------------------------- 1 | (* 2 | * Actor - Parallel & Distributed Engine of Owl System 3 | * Copyright (c) 2016-2018 Liang Wang 4 | *) 5 | 6 | (* Model Parallel: Parameter server module *) 7 | 8 | open Actor_types 9 | 10 | 11 | type param_context = Actor_types.param_context 12 | 13 | type barrier = ASP | BSP | SSP | PSP 14 | 15 | 16 | let start ?barrier jid url = 17 | (* reset the barrier control if specifed *) 18 | let _barrier_str = match barrier with 19 | | Some ASP -> Marshal.to_string Actor_barrier.param_asp [ Marshal.Closures ] 20 | | Some BSP -> Marshal.to_string Actor_barrier.param_bsp [ Marshal.Closures ] 21 | | Some SSP -> Marshal.to_string Actor_barrier.param_ssp [ Marshal.Closures ] 22 | | Some PSP -> failwith "actor_param:start:psp" 23 | | None -> Actor_paramserver.(!_barrier) 24 | in 25 | Actor_paramserver._barrier := _barrier_str; 26 | (* start preparing communication context *) 27 | let _ztx = ZMQ.Context.create () in 28 | let _addr, _router = Actor_utils.bind_available_addr _ztx in 29 | let req = ZMQ.Socket.create _ztx ZMQ.Socket.req in 30 | ZMQ.Socket.connect req url; 31 | Actor_utils.send req Job_Reg [|_addr; jid|]; 32 | (* create and initialise part of the context *) 33 | let _context = Actor_utils.empty_param_context () in 34 | _context.job_id <- jid; 35 | _context.myself_addr <- _addr; 36 | _context.myself_sock <- _router; 37 | _context.ztx <- _ztx; 38 | (* depends on the role, start server or client *) 39 | let m = of_msg (ZMQ.Socket.recv req) in 40 | let _ = match m.typ with 41 | | Job_Master -> Actor_paramserver.init m _context 42 | | Job_Worker -> Actor_paramclient.init m _context 43 | | _ -> Owl_log.info "%s" "unknown command"; 44 | in 45 | ZMQ.Socket.close req 46 | 47 | 48 | let register_barrier (f : ps_barrier_typ) = 49 | Actor_paramserver._barrier := Marshal.to_string f [ Marshal.Closures ] 50 | 51 | 52 | let register_schedule (f : ('a, 'b, 'c) ps_schedule_typ) = 53 | Actor_paramserver._schedule := Marshal.to_string f [ Marshal.Closures ] 54 | 55 | 56 | let register_pull (f : ('a, 'b, 'c) ps_pull_typ) = 57 | Actor_paramserver._pull := Marshal.to_string f [ Marshal.Closures ] 58 | 59 | 60 | let register_push (f : ('a, 'b, 'c) ps_push_typ) = 61 | Actor_paramclient._push := Marshal.to_string f [ Marshal.Closures ] 62 | 63 | 64 | let register_stop (f : ps_stop_typ) = 65 | Actor_paramserver._stop := Marshal.to_string f [ Marshal.Closures ] 66 | 67 | 68 | let get k = 69 | match Actor_paramserver.(!_context.job_id) = "" with 70 | | true -> Actor_paramclient._get k 71 | | false -> Actor_paramserver._get k 72 | 73 | 74 | let set k v = 75 | match Actor_paramserver.(!_context.job_id) = "" with 76 | | true -> Actor_paramclient.(_set k v !_context.step) 77 | | false -> Actor_paramserver.(_set k v !_context.step) 78 | 79 | 80 | let keys () = Hashtbl.fold (fun k _v l -> l @ [ Obj.obj k ]) Actor_paramserver._param [] 81 | 82 | 83 | let worker_num () = 84 | match Actor_paramserver.(!_context.job_id) = "" with 85 | | true -> failwith "actor_param:worker_num" 86 | | false -> StrMap.cardinal Actor_paramserver.(!_context.workers) 87 | -------------------------------------------------------------------------------- /lib/actor_param.mli: -------------------------------------------------------------------------------- 1 | (* 2 | * Actor - Parallel & Distributed Engine of Owl System 3 | * Copyright (c) 2016-2018 Liang Wang 4 | *) 5 | 6 | (* Model Parallel: Parameter server module *) 7 | 8 | open Actor_types 9 | 10 | (* context type, duplicate from Actor_types *) 11 | type param_context = Actor_types.param_context 12 | 13 | type barrier = 14 | | ASP (* Asynchronous Parallel *) 15 | | BSP (* Bulk Synchronous Parallel *) 16 | | SSP (* Stale Synchronous Parallel *) 17 | | PSP (* Probabilistic Synchronous Parallel *) 18 | 19 | 20 | (** core interfaces to parameter server *) 21 | 22 | val start : ?barrier:barrier -> string -> string -> unit 23 | (** start running the model loop *) 24 | 25 | val register_barrier : ps_barrier_typ -> unit 26 | (** register user-defined barrier function at p2p server *) 27 | 28 | val register_schedule : ('a, 'b, 'c) ps_schedule_typ -> unit 29 | (** register user-defined scheduler *) 30 | 31 | val register_pull : ('a, 'b, 'c) ps_pull_typ -> unit 32 | (** register user-defined pull function executed at master *) 33 | 34 | val register_push : ('a, 'b, 'c) ps_push_typ -> unit 35 | (** register user-defined push function executed at worker *) 36 | 37 | val register_stop : ps_stop_typ -> unit 38 | (** register stopping criterion function *) 39 | 40 | val get : 'a -> 'b * int 41 | (** given a key, get its value and timestamp *) 42 | 43 | val set : 'a -> 'b -> unit 44 | (** given a key, set its value at master *) 45 | 46 | val keys : unit -> 'a list 47 | (** FIXME: reture all the keys in a parameter server *) 48 | 49 | val worker_num : unit -> int 50 | (** return the number of workders, only work at server side *) 51 | -------------------------------------------------------------------------------- /lib/actor_param_sgd.ml: -------------------------------------------------------------------------------- 1 | (* 2 | * Actor - Parallel & Distributed Engine of Owl System 3 | * Copyright (c) 2016-2018 Liang Wang 4 | *) 5 | 6 | (* 7 | Distributed Stochastic Gradient Decendent 8 | 9 | Each row in the data matrix is a data point; 10 | each column in the model matrix is a classifier. 11 | *) 12 | 13 | open Owl 14 | open Actor_types 15 | 16 | module MX = Mat 17 | module PS = Actor_param 18 | 19 | 20 | (* variables used in distributed sgd *) 21 | let data_x = ref (MX.empty 0 0) 22 | let data_y = ref (MX.empty 0 0) 23 | let _model = ref (MX.empty 0 0) 24 | let gradfn = ref Owl_optimise.square_grad 25 | let lossfn = ref Owl_optimise.square_loss 26 | let step_t = ref 0.001 27 | 28 | 29 | (* prepare data, model, gradient, loss *) 30 | let init x y m g l = 31 | data_x := x; 32 | data_y := y; 33 | _model := m; 34 | gradfn := g; 35 | lossfn := l 36 | 37 | 38 | let calculate_gradient b x y m g l = 39 | let xt, i = MX.draw_rows x b in 40 | let yt = MX.rows y i in 41 | let yt' = MX.(xt *@ m) in 42 | let d = g xt yt yt' in 43 | Owl_log.debug "loss = %.10f" (l yt yt' |> MX.sum); 44 | d 45 | 46 | 47 | let schedule workers = 48 | let _, n = MX.shape !_model in 49 | List.map (fun x -> 50 | (* randomly choose a classifier *) 51 | let k = Stats.Rnd.uniform_int ~a:0 ~b:(n - 1) () in 52 | let v, _ = PS.get k in 53 | (x, [(k,v)]) 54 | ) workers 55 | 56 | 57 | let push id vars = 58 | (* update local model, need to improve for sparsity *) 59 | List.iter (fun (k,v) -> 60 | MX.copy_col_to v !_model k 61 | ) vars; 62 | (* compute the assigned work, return the update gradient *) 63 | (* TODO: lots to optimise, not compute whole model, cache local ... *) 64 | List.map (fun (k,v) -> 65 | let d = calculate_gradient 10 !data_x !data_y !_model !gradfn !lossfn in 66 | let d = MX.(d *$ !step_t) in 67 | (k, MX.col d k) 68 | ) vars 69 | 70 | 71 | let pull vars = 72 | List.map (fun (k,d) -> 73 | let v0, _ = PS.get k in 74 | let v1 = MX.(v0 - d) in 75 | (k,v1) 76 | ) vars 77 | 78 | 79 | let stop (_context : param_context ref) = !_context.step > 10_000 80 | 81 | 82 | let start jid = 83 | (* register schedule, push, pull functions *) 84 | PS.register_schedule schedule; 85 | PS.register_pull pull; 86 | PS.register_push push; 87 | PS.register_stop stop; 88 | (* pre-cache the model in the server's kv store *) 89 | (* FIXME: need to fix this hack *) 90 | MX.iteri_cols (fun k v -> Actor_paramserver._set k v 0) !_model; 91 | (* start running the ps *) 92 | Owl_log.info "PS: sdg algorithm starts running ..."; 93 | PS.start jid Actor_config.manager_addr 94 | -------------------------------------------------------------------------------- /lib/actor_paramclient.ml: -------------------------------------------------------------------------------- 1 | (* 2 | * Actor - Parallel & Distributed Engine of Owl System 3 | * Copyright (c) 2016-2018 Liang Wang 4 | *) 5 | 6 | (* Model Parallel: Parameter client module *) 7 | 8 | open Actor_types 9 | 10 | 11 | (* the global context: master, worker, etc. *) 12 | let _context = ref (Actor_utils.empty_param_context ()) 13 | 14 | 15 | (* default push function *) 16 | let _default_push = fun _worker_id _vars -> [] 17 | 18 | let _push = ref (Marshal.to_string _default_push [ Marshal.Closures ]) 19 | 20 | 21 | let _get k = 22 | let k' = Marshal.to_string k [] in 23 | Actor_utils.send ~bar:!_context.step !_context.master_sock PS_Get [|k'|]; 24 | let m = of_msg (ZMQ.Socket.recv ~block:true !_context.master_sock) in 25 | Marshal.from_string m.par.(0) 0, m.bar 26 | 27 | 28 | let _set k v t = 29 | let k' = Marshal.to_string k [] in 30 | let v' = Marshal.to_string v [] in 31 | Actor_utils.send ~bar:t !_context.master_sock PS_Set [|k'; v'|] 32 | 33 | 34 | let update_param x t = 35 | (* update multiple kvs, more efficient than set *) 36 | let x' = Marshal.to_string x [] in 37 | Actor_utils.send ~bar:t !_context.master_sock PS_Push [|x'|] 38 | 39 | 40 | let service_loop () = 41 | Owl_log.debug "parameter worker @ %s" !_context.myself_addr; 42 | (* unmarshal the push function *) 43 | let push : 'a -> ('b * 'c) list -> ('b * 'c) list = Marshal.from_string !_push 0 in 44 | (* loop to process messages *) 45 | try while true do 46 | let _i, m = Actor_utils.recv !_context.myself_sock in 47 | let t = m.bar in 48 | match m.typ with 49 | | PS_Schedule -> ( 50 | Owl_log.debug "%s: ps_schedule" !_context.myself_addr; 51 | !_context.step <- (if t > !_context.step then t else !_context.step); 52 | let vars = Marshal.from_string m.par.(0) 0 in 53 | let updates = push !_context.myself_addr vars in 54 | update_param updates t 55 | ) 56 | | Terminate -> ( 57 | Owl_log.debug "%s: terminate"!_context.myself_addr; 58 | Actor_utils.send ~bar:t !_context.master_sock OK [||]; 59 | Unix.sleep 1; (* FIXME: sleep ... *) 60 | failwith ("#" ^ !_context.job_id ^ " terminated") 61 | ) 62 | | _ -> ( Owl_log.debug "unknown mssage to PS" ) 63 | done with Failure e -> ( 64 | Owl_log.warn "%s" e; 65 | ZMQ.Socket.close !_context.myself_sock; 66 | Pervasives.exit 0 ) 67 | 68 | 69 | let init m context = 70 | _context := context; 71 | !_context.master_addr <- m.par.(0); 72 | (* connect to job master *) 73 | let master = ZMQ.Socket.create !_context.ztx ZMQ.Socket.dealer in 74 | ZMQ.Socket.set_send_high_water_mark master Actor_config.high_warter_mark; 75 | ZMQ.Socket.set_identity master !_context.myself_addr; 76 | ZMQ.Socket.connect master !_context.master_addr; 77 | Actor_utils.send master OK [|!_context.myself_addr|]; 78 | !_context.master_sock <- master; 79 | (* enter into worker service loop *) 80 | service_loop () 81 | -------------------------------------------------------------------------------- /lib/actor_paramserver.ml: -------------------------------------------------------------------------------- 1 | (* 2 | * Actor - Parallel & Distributed Engine of Owl System 3 | * Copyright (c) 2016-2018 Liang Wang 4 | *) 5 | 6 | (* Model Parallel: Parameter server module *) 7 | 8 | open Actor_types 9 | 10 | 11 | (* the global context: master, worker, etc. *) 12 | let _context = ref (Actor_utils.empty_param_context ()) 13 | 14 | let _param : (Obj.t, Obj.t * int) Hashtbl.t = Hashtbl.create 1_000_000 15 | 16 | 17 | (* default schedule function *) 18 | let _default_schedule = fun _ -> [ ] (** TODO: fix scheduler ... *) 19 | 20 | let _schedule = ref (Marshal.to_string _default_schedule [ Marshal.Closures ]) 21 | 22 | 23 | (* default pull function *) 24 | let _default_pull = fun updates -> updates 25 | 26 | let _pull = ref (Marshal.to_string _default_pull [ Marshal.Closures ]) 27 | 28 | 29 | (* default stopping function *) 30 | let _default_stop = fun _ -> false 31 | 32 | let _stop = ref (Marshal.to_string _default_stop [ Marshal.Closures ]) 33 | 34 | 35 | (* default barrier function *) 36 | let _default_barrier = Actor_barrier.param_bsp 37 | 38 | let _barrier = ref (Marshal.to_string _default_barrier [ Marshal.Closures ]) 39 | 40 | 41 | let update_steps t w = 42 | let t' = Hashtbl.find !_context.worker_step w in 43 | match t > t' with 44 | | true -> ( 45 | Hashtbl.replace !_context.worker_busy w 0; 46 | Hashtbl.replace !_context.worker_step w t; 47 | Hashtbl.add !_context.step_worker t w ) 48 | | false -> () 49 | 50 | 51 | let _get k = 52 | let k' = Obj.repr k in 53 | let v, t = Hashtbl.find _param k' in 54 | Obj.obj v, t 55 | 56 | 57 | let _set k v t = 58 | let k' = Obj.repr k in 59 | let v' = Obj.repr v in 60 | match Hashtbl.mem _param k' with 61 | | true -> Hashtbl.replace _param k' (v',t) 62 | | false -> Hashtbl.add _param k' (v',t) 63 | 64 | 65 | let _broadcast_all t s = 66 | StrMap.iter (fun _k v -> Actor_utils.send ~bar:!_context.step v t s) !_context.workers; 67 | !_context.step 68 | 69 | 70 | let terminate () = 71 | let _ = _broadcast_all Terminate [||] in 72 | Unix.sleep 1 (** FIXME: change to BSP *) 73 | 74 | 75 | let service_loop () = 76 | Owl_log.debug "parameter server @ %s" !_context.myself_addr; 77 | (* unmarshal the schedule and pull functions *) 78 | let schedule : ('a, 'b, 'c) ps_schedule_typ = Marshal.from_string !_schedule 0 in 79 | let pull : ('a, 'b, 'c) ps_pull_typ = Marshal.from_string !_pull 0 in 80 | let barrier : ps_barrier_typ = Marshal.from_string !_barrier 0 in 81 | let stop : ps_stop_typ = Marshal.from_string !_stop 0 in 82 | (* loop to process messages *) 83 | try while not (stop _context) do 84 | (* synchronisation barrier check *) 85 | let t, passed = barrier _context in !_context.step <- t; 86 | (* schecule the passed at every message arrival *) 87 | let tasks = schedule passed in 88 | List.iter (fun (worker, task) -> 89 | let w = StrMap.find worker !_context.workers in 90 | let s = Marshal.to_string task [] in 91 | let t = Hashtbl.find !_context.worker_step worker + 1 in 92 | let _ = Hashtbl.replace !_context.worker_busy worker 1 in 93 | Actor_utils.send ~bar:t w PS_Schedule [|s|] 94 | ) tasks; 95 | if List.length tasks > 0 then 96 | Owl_log.debug "schedule t:%i -> %i workers" !_context.step (List.length tasks); 97 | (** wait for another message arrival *) 98 | let i, m = Actor_utils.recv !_context.myself_sock in 99 | let t = m.bar in 100 | match m.typ with 101 | | PS_Get -> ( 102 | Owl_log.debug "%s: ps_get" !_context.myself_addr; 103 | let k = Marshal.from_string m.par.(0) 0 in 104 | let v, t' = _get k in 105 | let s = to_msg t' OK [| Marshal.to_string v [] |] in 106 | ZMQ.Socket.send_all ~block:false !_context.myself_sock [i;s] 107 | ) 108 | | PS_Set -> ( 109 | Owl_log.debug "%s: ps_set" !_context.myself_addr; 110 | let k = Marshal.from_string m.par.(0) 0 in 111 | let v = Marshal.from_string m.par.(1) 0 in 112 | _set k v t 113 | ) 114 | | PS_Push -> ( 115 | Owl_log.debug "%s: ps_push" !_context.myself_addr; 116 | let updates = Marshal.from_string m.par.(0) 0 |> pull in 117 | List.iter (fun (k,v) -> _set k v t) updates; 118 | update_steps t i 119 | ) 120 | | _ -> Owl_log.debug "unknown mssage to PS" 121 | done with Failure e -> ( 122 | Owl_log.warn "%s" e; 123 | terminate (); 124 | ZMQ.Socket.close !_context.myself_sock ) 125 | 126 | 127 | let init m context = 128 | _context := context; 129 | (* contact allocated actors to assign jobs *) 130 | let addrs = Marshal.from_string m.par.(0) 0 in 131 | List.map (fun x -> 132 | let req = ZMQ.Socket.create !_context.ztx ZMQ.Socket.req in 133 | ZMQ.Socket.connect req x; 134 | let app = Filename.basename Sys.argv.(0) in 135 | let arg = Marshal.to_string Sys.argv [] in 136 | Actor_utils.send req Job_Create [|!_context.myself_addr; app; arg|]; req 137 | ) addrs 138 | |> List.iter ZMQ.Socket.close; 139 | (* wait until all the allocated actors register *) 140 | while (StrMap.cardinal !_context.workers) < (List.length addrs) do 141 | let _i, m = Actor_utils.recv !_context.myself_sock in 142 | let s = ZMQ.Socket.create !_context.ztx ZMQ.Socket.dealer in 143 | ZMQ.Socket.set_send_high_water_mark s Actor_config.high_warter_mark; 144 | ZMQ.Socket.connect s m.par.(0); 145 | !_context.workers <- (StrMap.add m.par.(0) s !_context.workers); 146 | done; 147 | (* initialise the step <--> work tables *) 148 | StrMap.iter (fun k _v -> 149 | Hashtbl.add !_context.worker_busy k 0; 150 | Hashtbl.add !_context.worker_step k 0; 151 | Hashtbl.add !_context.step_worker 0 k; 152 | ) !_context.workers; 153 | (* enter into master service loop *) 154 | service_loop () 155 | -------------------------------------------------------------------------------- /lib/actor_peer.ml: -------------------------------------------------------------------------------- 1 | (* 2 | * Actor - Parallel & Distributed Engine of Owl System 3 | * Copyright (c) 2016-2018 Liang Wang 4 | *) 5 | 6 | (* Peer-to-Peer Parallel *) 7 | 8 | open Actor_types 9 | 10 | 11 | let start jid url = 12 | let _ztx = ZMQ.Context.create () in 13 | let _addr, _router = Actor_utils.bind_available_addr _ztx in 14 | let req = ZMQ.Socket.create _ztx ZMQ.Socket.req in 15 | ZMQ.Socket.connect req url; 16 | Actor_utils.send req P2P_Reg [|_addr; jid|]; 17 | (* create and initialise part of the context *) 18 | let _context = Actor_utils.empty_peer_context () in 19 | _context.job_id <- jid; 20 | _context.myself_addr <- _addr; 21 | _context.myself_sock <- _router; 22 | _context.ztx <- _ztx; 23 | (* equivalent role, client is a new process *) 24 | let m = of_msg (ZMQ.Socket.recv req) in 25 | let _ = match m.typ with 26 | | OK -> ( 27 | match Unix.fork () with 28 | | 0 -> Actor_peerclient.init m _context 29 | | _p -> Actor_peerserver.init m _context 30 | ) 31 | | _ -> Owl_log.info "start: unknown command" 32 | in 33 | ZMQ.Socket.close req 34 | 35 | 36 | (* basic architectural functions for p2p parallel *) 37 | 38 | let register_barrier (f : p2p_barrier_typ) = 39 | Actor_peerserver._barrier := Marshal.to_string f [ Marshal.Closures ] 40 | 41 | 42 | let register_pull (f : ('a, 'b) p2p_pull_typ) = 43 | Actor_peerserver._pull := Marshal.to_string f [ Marshal.Closures ] 44 | 45 | 46 | let register_schedule (f : 'a p2p_schedule_typ) = 47 | Actor_peerclient._schedule := Marshal.to_string f [ Marshal.Closures ] 48 | 49 | 50 | let register_push (f : ('a, 'b) p2p_push_typ) = 51 | Actor_peerclient._push := Marshal.to_string f [ Marshal.Closures ] 52 | 53 | 54 | let register_stop (f : p2p_stop_typ) = 55 | Actor_peerclient._stop := Marshal.to_string f [ Marshal.Closures ] 56 | 57 | 58 | (* some helper functions for various strategies *) 59 | 60 | let is_server () = Actor_peerclient.(!_context.job_id) = "" 61 | 62 | 63 | let get k = 64 | match is_server () with 65 | | true -> Actor_peerserver._get k 66 | | false -> Actor_peerclient._get k 67 | 68 | 69 | let set k v = 70 | match is_server () with 71 | | true -> Actor_peerserver.(_set k v !_context.step) 72 | | false -> Actor_peerclient.(_set k v) 73 | 74 | 75 | let _swarm_size = None 76 | -------------------------------------------------------------------------------- /lib/actor_peer.mli: -------------------------------------------------------------------------------- 1 | (* 2 | * Actor - Parallel & Distributed Engine of Owl System 3 | * Copyright (c) 2016-2018 Liang Wang 4 | *) 5 | 6 | (* Peer-to-Peer Parallel *) 7 | 8 | open Actor_types 9 | 10 | (** start running the model loop *) 11 | val start : string -> string -> unit 12 | 13 | (** register user-defined barrier function at p2p server *) 14 | val register_barrier : p2p_barrier_typ -> unit 15 | 16 | (** register user-defined pull function at p2p server *) 17 | val register_pull : ('a, 'b) p2p_pull_typ -> unit 18 | 19 | (** register user-defined scheduler at p2p client *) 20 | val register_schedule : 'a p2p_schedule_typ -> unit 21 | 22 | (** register user-defined push function at p2p client *) 23 | val register_push : ('a, 'b) p2p_push_typ -> unit 24 | 25 | (** register stopping criterion function at p2p client *) 26 | val register_stop : p2p_stop_typ -> unit 27 | 28 | (** given a key, get its value and timestamp *) 29 | val get : 'a -> 'b * int 30 | 31 | (** given a key, set its value at master *) 32 | val set : 'a -> 'b -> unit 33 | -------------------------------------------------------------------------------- /lib/actor_peer_lda.ml: -------------------------------------------------------------------------------- 1 | (* 2 | * Actor - Parallel & Distributed Engine of Owl System 3 | * Copyright (c) 2016-2018 Liang Wang 4 | *) 5 | 6 | (* Peer LDA: Latent Dirichlet Allocation *) 7 | 8 | open Owl 9 | open Actor_types 10 | 11 | 12 | module MS = Sparse.Dok_matrix 13 | module MD = Mat 14 | module P2P = Actor_peer 15 | 16 | 17 | (* local model hyper-parameters *) 18 | let alpha = ref 0. 19 | let beta = ref 0. 20 | let alpha_k = ref 0. 21 | let beta_v = ref 0. 22 | (* local and global model variables *) 23 | let t_dk = ref (MD.zeros 1 1) 24 | let t_wk = ref (MD.zeros 1 1) 25 | let t__k = ref (MD.zeros 1 1) 26 | let t__z = ref [| [||] |] 27 | (* data set variables *) 28 | let n_d = ref 0 29 | let n_k = ref 0 30 | let n_v = ref 0 31 | let data = ref [| [||] |] 32 | let vocb : (string, int) Hashtbl.t ref = ref (Hashtbl.create 1) 33 | let b__m = ref [||] (* track if local model has been merged *) 34 | 35 | 36 | let include_token w d k = 37 | MD.(set !t__k 0 k (get !t__k 0 k +. 1.)); 38 | MD.(set !t_wk w k (get !t_wk w k +. 1.)); 39 | MD.(set !t_dk d k (get !t_dk d k +. 1.)) 40 | 41 | 42 | let exclude_token w d k = 43 | MD.(set !t__k 0 k (get !t__k 0 k -. 1.)); 44 | MD.(set !t_wk w k (get !t_wk w k -. 1.)); 45 | MD.(set !t_dk d k (get !t_dk d k -. 1.)) 46 | 47 | 48 | let init k v d = 49 | Log.info "init the model"; 50 | data := d; 51 | vocb := v; 52 | b__m := Array.make (Hashtbl.length v) false; 53 | (* set model parameters *) 54 | n_d := Array.length d; 55 | n_v := Hashtbl.length v; 56 | n_k := k; 57 | t_dk := MD.zeros !n_d !n_k; 58 | t_wk := MD.zeros !n_v !n_k; 59 | t__k := MD.zeros 1 !n_k; 60 | (* set model hyper-parameters *) 61 | alpha := 50.; 62 | alpha_k := !alpha /. (float_of_int !n_k); 63 | beta := 0.1; 64 | beta_v := (float_of_int !n_v) *. !beta; 65 | (* randomise the topic assignment for each token *) 66 | t__z := Array.mapi (fun i s -> 67 | Array.init (Array.length s) (fun j -> 68 | let k' = Stats.uniform_int_rvs ~a:0 ~b:(k - 1) in 69 | include_token s.(j) i k'; 70 | k' 71 | ) 72 | ) d; 73 | (* init local model / kv store *) 74 | for w = 0 to !n_v - 1 do P2P.set w [||] done; 75 | P2P.set (-1) !t__k 76 | 77 | 78 | let rebuild_local_model () = 79 | Owl_log.warn "rebuild local model start"; 80 | t_wk := MD.zeros !n_v !n_k; 81 | Array.iteri (fun i d -> 82 | Array.iteri (fun j k -> 83 | let w = !data.(i).(j) in 84 | MD.(set !t_wk w k (get !t_wk w k +. 1.)); 85 | ) d 86 | ) !t__z; 87 | Owl_log.warn "rebuild local model finished" 88 | 89 | 90 | let show_stats () = 91 | Owl_log.info "t_wk = %.4f, t_dk = %.4f" (MD.density !t_wk) (MD.density !t_dk) 92 | 93 | 94 | let sampling d h = 95 | let p = MD.zeros 1 !n_k in 96 | Array.iteri (fun i w -> 97 | if h.(w) = true then ( 98 | let k = !t__z.(d).(i) in 99 | exclude_token w d k; 100 | (* make cdf function *) 101 | let x = ref 0. in 102 | for j = 0 to !n_k - 1 do 103 | x := !x +. (MD.get !t_dk d j +. !alpha_k) *. (MD.get !t_wk w j +. !beta) /. (MD.get !t__k 0 j +. !beta_v); 104 | MD.set p 0 j !x 105 | done; 106 | (* draw a sample *) 107 | let u = Stats.uniform_rvs ~a:0. ~b:1. *. !x in 108 | let k = ref 0 in 109 | while (MD.get p 0 !k) < u do k := !k + 1; done; 110 | include_token w d !k; 111 | !t__z.(d).(i) <- !k; 112 | ) 113 | ) !data.(d) 114 | 115 | 116 | let schedule _context = 117 | Owl_log.info "schedule @ %s, step:%i" !_context.master_addr !_context.step; 118 | let d = Array.init !n_v (fun i -> i) in 119 | Stats.choose d (!n_v / 10) |> Array.to_list 120 | 121 | 122 | let pull _context updates = 123 | let num_updates = List.fold_right (fun (_,a,_) x -> Array.length a + x) updates 0 in 124 | Owl_log.info "pull @ %s, updates:%i" !_context.myself_addr num_updates; 125 | (* update t__k *) 126 | let tk_updates = List.filter (fun (w,_,_) -> w = -1) updates in 127 | if List.length tk_updates > 0 then ( 128 | let t_k', _ = P2P.get (-1) in 129 | Owl_log.error "%s ==> %i %f" !_context.myself_addr (List.length tk_updates) (MD.sum' t_k'); 130 | List.iter (fun (_,a,_t) -> 131 | Array.iter (fun (k,c) -> MD.(set t_k' 0 k (get t_k' 0 k +. c))) a 132 | ) tk_updates; 133 | P2P.set (-1) t_k' 134 | ); 135 | (* update t_wk *) 136 | let wk_updates = List.filter (fun (w,_,_) -> w > -1) updates in 137 | let h = Hashtbl.create 256 in 138 | List.iter (fun (w,a,t) -> 139 | if Hashtbl.mem h w = false then ( 140 | let r = MD.zeros 1 !n_k in 141 | let a', t' = P2P.get w in 142 | Array.iter (fun (k,c) -> MD.set r 0 k c) a'; 143 | Hashtbl.add h w (r,t') 144 | ); 145 | let r, t' = Hashtbl.find h w in 146 | Array.iter (fun (k,c) -> MD.(set r 0 k (get r 0 k +. c))) a; 147 | Hashtbl.replace h w (r, max t t'); 148 | ) wk_updates; 149 | let wk_updates' = ref [] in 150 | Hashtbl.iter (fun w (r,t) -> 151 | let a = Array.make (MD.nnz r) (0,0.) in 152 | let j = ref 0 in 153 | MD.iteri (fun k c -> 154 | if c <> 0. then (a.(!j) <- (k,c); j := !j + 1) 155 | ) r; 156 | wk_updates' := !wk_updates' @ [(w,a,t)] 157 | ) h; 158 | !wk_updates' 159 | 160 | 161 | let push _context params = 162 | Owl_log.info "push @ %s" !_context.master_addr; 163 | show_stats (); 164 | (* a workaround for t__k at the moment *) 165 | let t_k' = P2P.get (-1) |> fst in 166 | t__k := MD.copy t_k'; 167 | (* if there are words to be merged into global model, rebuild local one *) 168 | let shall_rebuild = ref false in 169 | Array.iter (fun b -> if b = false then shall_rebuild := true) !b__m; 170 | if !shall_rebuild then rebuild_local_model (); 171 | (* update local model and set bitmap of words *) 172 | let h = Array.make !n_v false in 173 | List.iteri (fun _i (w,a) -> 174 | if !b__m.(w) = true then ( 175 | Array.iter (fun (k,c) -> MD.set !t_wk w k c) a 176 | ) 177 | else ( 178 | Array.iter (fun (k,c) -> 179 | MD.(set !t_wk w k (get !t_wk w k +. c)); 180 | MD.(set !t__k 0 k (get !t__k 0 k +. c)) 181 | ) a; 182 | !b__m.(w) <- true 183 | ); 184 | h.(w) <- true; 185 | ) params; 186 | (* iterate all local docs, time-consuming *) 187 | for j = 0 to !n_d - 1 do sampling j h done; 188 | (* calculate model updates *) 189 | let updates = ref [] in 190 | List.iter (fun (w,a) -> 191 | let h = Array.make !n_k false in 192 | let a' = ref [||] in 193 | Array.iter (fun (k,c) -> 194 | h.(k) <- true; 195 | let c' = MD.get !t_wk w k in 196 | if c' <> c then a' := Array.append !a' [|(k,c'-.c)|] 197 | ) a; 198 | let r = MD.row !t_wk w in 199 | MD.iteri (fun k c' -> 200 | if h.(k) = false && c' <> 0. then a' := Array.append !a' [|(k,c')|] 201 | ) r; 202 | updates := !updates @ [(w,!a')] 203 | ) params; 204 | (* calculate t__k updates *) 205 | let a = ref [||] in 206 | MD.iteri (fun k c -> 207 | if c <> 0. then a := Array.append !a [|(k,c)|] 208 | ) MD.(!t__k - t_k'); 209 | updates := !updates @ [(-1,!a)]; 210 | !updates 211 | 212 | 213 | let barrier _context = Actor_barrier.p2p_bsp _context 214 | 215 | 216 | let stop _context = !_context.step > 5_00 217 | 218 | 219 | let start jid = 220 | (* register schedule, push, pull functions *) 221 | P2P.register_barrier barrier; 222 | P2P.register_schedule schedule; 223 | P2P.register_push push; 224 | P2P.register_pull pull; 225 | P2P.register_stop stop; 226 | (* start running the ps *) 227 | Owl_log.info "P2P: lda algorithm starts running ..."; 228 | P2P.start jid Actor_config.manager_addr 229 | -------------------------------------------------------------------------------- /lib/actor_peer_sgd0.ml: -------------------------------------------------------------------------------- 1 | (* 2 | * Actor - Parallel & Distributed Engine of Owl System 3 | * Copyright (c) 2016-2018 Liang Wang 4 | *) 5 | 6 | (* Distributed Stochastic Gradient Descendent *) 7 | 8 | open Owl 9 | open Actor_types 10 | 11 | 12 | module MX = Mat 13 | module P2P = Actor_peer 14 | 15 | 16 | (* variables used in distributed sgd *) 17 | let data_x = ref (MX.empty 0 0) 18 | let data_y = ref (MX.empty 0 0) 19 | let _model = ref (MX.empty 0 0) 20 | let gradfn = ref Owl_optimise.square_grad 21 | let lossfn = ref Owl_optimise.square_loss 22 | let step_t = ref 0.001 23 | 24 | 25 | (* prepare data, model, gradient, loss *) 26 | let init x y m g l = 27 | data_x := x; 28 | data_y := y; 29 | _model := m; 30 | gradfn := g; 31 | lossfn := l; 32 | MX.iteri_cols (fun i v -> P2P.set i v) !_model 33 | 34 | 35 | let calculate_gradient b x y m g l = 36 | let xt, i = MX.draw_rows x b in 37 | let yt = MX.rows y i in 38 | let yt' = MX.(xt *@ m) in 39 | let d = g xt yt yt' in 40 | Owl_log.debug "loss = %.10f" (l yt yt' |> MX.sum); 41 | d 42 | 43 | 44 | let schedule _context = 45 | Owl_log.debug "%s: scheduling ..." !_context.master_addr; 46 | let n = MX.col_num !_model in 47 | let k = Stats.Rnd.uniform_int ~a:0 ~b:(n - 1) () in 48 | [ k ] 49 | 50 | 51 | let push _context params = 52 | List.map (fun (k,v) -> 53 | Owl_log.debug "%s: working on %i ..." !_context.master_addr k; 54 | let y = MX.col !data_y k in 55 | let d = calculate_gradient 10 !data_x y v !gradfn !lossfn in 56 | let d = MX.(d *$ !step_t) in 57 | (k, d) 58 | ) params 59 | 60 | 61 | let barrier _context = 62 | Owl_log.debug "checking barrier ..."; 63 | true 64 | 65 | 66 | let pull _context updates = 67 | Owl_log.debug "pulling updates ..."; 68 | List.map (fun (k,v,t) -> 69 | let v0, _ = P2P.get k in 70 | let v1 = MX.(v0 - v) in 71 | k, v1, t 72 | ) updates 73 | 74 | 75 | let stop _context = false 76 | 77 | 78 | let start jid = 79 | (* register schedule, push, pull functions *) 80 | P2P.register_barrier barrier; 81 | P2P.register_schedule schedule; 82 | P2P.register_push push; 83 | P2P.register_pull pull; 84 | P2P.register_stop stop; 85 | (* start running the ps *) 86 | Owl_log.info "P2P: sdg algorithm starts running ..."; 87 | P2P.start jid Actor_config.manager_addr 88 | -------------------------------------------------------------------------------- /lib/actor_peer_sgd1.ml: -------------------------------------------------------------------------------- 1 | (* 2 | * Actor - Parallel & Distributed Engine of Owl System 3 | * Copyright (c) 2016-2018 Liang Wang 4 | *) 5 | 6 | (* Distributed Stochastic Gradient Decendent *) 7 | 8 | open Owl 9 | open Actor_types 10 | 11 | 12 | module MX = Mat 13 | module P2P = Actor_peer 14 | 15 | 16 | (* variables used in distributed sgd *) 17 | let data_x = ref (MX.empty 0 0) 18 | let data_y = ref (MX.empty 0 0) 19 | let _model = ref (MX.empty 0 0) 20 | let gradfn = ref Owl_optimise.square_grad 21 | let lossfn = ref Owl_optimise.square_loss 22 | let step_t = ref 0.001 23 | 24 | 25 | (* prepare data, model, gradient, loss *) 26 | let init x y m g l = 27 | data_x := x; 28 | data_y := y; 29 | _model := m; 30 | gradfn := g; 31 | lossfn := l; 32 | MX.iteri_cols (fun i v -> P2P.set i v) !_model 33 | 34 | 35 | let calculate_gradient b x y m g l = 36 | let xt, i = MX.draw_rows x b in 37 | let yt = MX.rows y i in 38 | let yt' = MX.(xt *@ m) in 39 | let d = g xt yt yt' in 40 | Owl_log.info "loss = %.10f" (l yt yt' |> MX.sum); 41 | d 42 | 43 | 44 | let update_local_model = None 45 | 46 | 47 | let schedule _context = 48 | let n = MX.col_num !_model in 49 | let k = Stats.Rnd.uniform_int ~a:0 ~b:(n - 1) () in 50 | [ k ] 51 | 52 | 53 | let push _context params = 54 | List.map (fun (k,v) -> 55 | let y = MX.col !data_y k in 56 | let d = calculate_gradient 10 !data_x y v !gradfn !lossfn in 57 | let d = MX.(d *$ !step_t) in 58 | (k, d) 59 | ) params 60 | 61 | 62 | let barrier _context = Actor_barrier.p2p_bsp _context 63 | 64 | 65 | let pull _context updates = 66 | let h = Hashtbl.create 32 in 67 | List.iter (fun (k,v,t) -> 68 | if Hashtbl.mem h k = false then ( 69 | let v', t' = P2P.get k in 70 | Hashtbl.add h k (v',t') 71 | ); 72 | let v', t' = Hashtbl.find h k in 73 | let v' = MX.(v' - v) in 74 | let t' = max t' t in 75 | Hashtbl.replace h k (v',t') 76 | ) updates; 77 | Hashtbl.fold (fun k (v,t) l -> l @ [(k,v,t)]) h [] 78 | 79 | 80 | let stop _context = !_context.step > 10_000 81 | 82 | 83 | let start jid = 84 | (* register schedule, push, pull functions *) 85 | P2P.register_barrier barrier; 86 | P2P.register_schedule schedule; 87 | P2P.register_push push; 88 | P2P.register_pull pull; 89 | P2P.register_stop stop; 90 | (* start running the ps *) 91 | Owl_log.info "P2P: sdg algorithm starts running ..."; 92 | P2P.start jid Actor_config.manager_addr 93 | -------------------------------------------------------------------------------- /lib/actor_peerclient.ml: -------------------------------------------------------------------------------- 1 | (* 2 | * Actor - Parallel & Distributed Engine of Owl System 3 | * Copyright (c) 2016-2018 Liang Wang 4 | *) 5 | 6 | (* Peer-to-Peer Parallel: Client module *) 7 | 8 | open Actor_types 9 | 10 | 11 | (* the global context: master, worker, etc. *) 12 | let _context = ref (Actor_utils.empty_peer_context ()) 13 | 14 | 15 | (* default schedule function *) 16 | let _default_schedule = fun _ -> [ ] 17 | 18 | let _schedule = ref (Marshal.to_string _default_schedule [ Marshal.Closures ]) 19 | 20 | 21 | (* default push function *) 22 | let _default_push = fun _ _ -> [] 23 | 24 | let _push = ref (Marshal.to_string _default_push [ Marshal.Closures ]) 25 | 26 | 27 | (* default stopping function *) 28 | let _default_stop = fun _ -> false 29 | 30 | let _stop = ref (Marshal.to_string _default_stop [ Marshal.Closures ]) 31 | 32 | 33 | let _get k = 34 | let k = Marshal.to_string k [] in 35 | let s = [|k; !_context.master_addr|] in 36 | Actor_utils.send !_context.master_sock P2P_Get s; 37 | let _, m = Actor_utils.recv !_context.myself_sock in 38 | let _k, v, t = Marshal.from_string m.par.(0) 0 in 39 | v, t 40 | 41 | 42 | let _set k v = 43 | let s = Marshal.to_string (k, v, -1) [] in 44 | Actor_utils.send !_context.master_sock P2P_Set [|s|] 45 | 46 | 47 | let _push_model params = 48 | let s = Marshal.to_string params [] in 49 | Actor_utils.send !_context.master_sock P2P_Push [|s|] 50 | 51 | 52 | let _pull_model params = 53 | List.map (fun k -> let v, _ = _get k in (k,v)) params 54 | 55 | 56 | let _pull_model_batch params = 57 | let s = Marshal.to_string params [] in 58 | Actor_utils.send !_context.master_sock P2P_Pull [|s|]; 59 | let _, m = Actor_utils.recv !_context.myself_sock in 60 | let kvs = Marshal.from_string m.par.(0) 0 in 61 | kvs 62 | 63 | 64 | let _barrier () = 65 | Actor_utils.send !_context.master_sock P2P_Bar [||]; 66 | let _, m = Actor_utils.recv !_context.myself_sock in 67 | !_context.step <- m.bar 68 | 69 | 70 | let service_loop () = 71 | Owl_log.debug "p2p_client @ %s" !_context.master_addr; 72 | (* unmarshal the schedule and push function *) 73 | let schedule : 'a p2p_schedule_typ = Marshal.from_string !_schedule 0 in 74 | let push : ('a, 'b) p2p_push_typ = Marshal.from_string !_push 0 in 75 | let stop : p2p_stop_typ = Marshal.from_string !_stop 0 in 76 | (* loop to process messages *) 77 | try while not (stop _context) do 78 | schedule _context 79 | |> _pull_model 80 | |> push _context 81 | |> _push_model 82 | |> _barrier 83 | done with Failure e -> ( 84 | Owl_log.warn "%s" e; 85 | ZMQ.Socket.close !_context.myself_sock; 86 | Pervasives.exit 0 ) 87 | 88 | 89 | let init _m context = 90 | _context := context; 91 | (* re-initialise since it is a new process *) 92 | !_context.ztx <- ZMQ.Context.create (); 93 | !_context.master_addr <- context.myself_addr; 94 | let _addr, _router = Actor_utils.bind_available_addr !_context.ztx in 95 | !_context.myself_addr <- _addr; 96 | !_context.myself_sock <- _router; 97 | (* set up local p2p server <-> client *) 98 | let sock = ZMQ.Socket.create !_context.ztx ZMQ.Socket.dealer in 99 | ZMQ.Socket.set_send_high_water_mark sock Actor_config.high_warter_mark; 100 | ZMQ.Socket.set_identity sock !_context.myself_addr; 101 | ZMQ.Socket.connect sock !_context.master_addr; 102 | !_context.master_sock <- sock; 103 | Actor_utils.send !_context.master_sock P2P_Connect [|!_context.myself_addr|]; 104 | (* enter into client service loop *) 105 | service_loop () 106 | -------------------------------------------------------------------------------- /lib/actor_peerserver.ml: -------------------------------------------------------------------------------- 1 | (* 2 | * Actor - Parallel & Distributed Engine of Owl System 3 | * Copyright (c) 2016-2018 Liang Wang 4 | *) 5 | 6 | (* Peer-to-Peer Parallel: Server module *) 7 | 8 | open Actor_types 9 | 10 | 11 | (* the global context: master, worker, etc. *) 12 | let _context = ref (Actor_utils.empty_peer_context ()) 13 | 14 | let _param : (Obj.t, Obj.t * int) Hashtbl.t = Hashtbl.create 1_000_000 15 | 16 | 17 | (* buffer of the requests and replies of pulling model parameters *) 18 | let _plbuf : (Obj.t, Obj.t option) Hashtbl.t = Hashtbl.create 1_000 19 | 20 | 21 | (* default pull function *) 22 | let _default_pull _ updates = updates 23 | 24 | let _pull = ref (Marshal.to_string _default_pull [ Marshal.Closures ]) 25 | 26 | 27 | (* default barrier function *) 28 | let _default_barrier = fun _ -> true 29 | 30 | let _barrier = ref (Marshal.to_string _default_barrier [ Marshal.Closures ]) 31 | 32 | 33 | (* routing table module *) 34 | module Route = struct 35 | 36 | (* FIXME: given ONLY 30-bit address space *) 37 | let _space = 2. ** 30. |> int_of_float 38 | 39 | 40 | let hash x = Hashtbl.hash x 41 | 42 | 43 | let distance x y = (x - y + _space) mod _space 44 | 45 | 46 | let add addr sock = 47 | if Hashtbl.mem !_context.spbuf addr = false then Hashtbl.add !_context.spbuf addr 0; 48 | !_context.workers <- (StrMap.add addr sock !_context.workers) 49 | 50 | 51 | let exists addr = StrMap.mem addr !_context.workers 52 | 53 | 54 | let connect addr = 55 | let sock = ZMQ.Socket.create !_context.ztx ZMQ.Socket.dealer in 56 | ZMQ.Socket.set_send_high_water_mark sock Actor_config.high_warter_mark; 57 | ZMQ.Socket.set_identity sock !_context.myself_addr; 58 | ZMQ.Socket.connect sock addr; 59 | sock 60 | 61 | 62 | let furthest x = 63 | let d = ref min_int in 64 | let n = ref "" in 65 | List.iteri (fun _i y -> 66 | let d' = distance (hash y) x in 67 | if d' > !d then ( d := d'; n := y ) 68 | ) (StrMap.keys !_context.workers @ [!_context.myself_addr]); 69 | !n 70 | 71 | 72 | let furthest_exclude x l = 73 | let addrs = StrMap.keys !_context.workers @ [!_context.myself_addr] 74 | |> List.filter (fun x -> not (List.mem x l)) 75 | in 76 | let d = ref min_int in 77 | let n = ref "" in 78 | List.iteri (fun _i y -> 79 | let d' = distance (hash y) x in 80 | if d' > !d then ( d := d'; n := y ) 81 | ) addrs; 82 | !n 83 | 84 | 85 | let nearest x = 86 | let d = ref max_int in 87 | let n = ref "" in 88 | List.iteri (fun _i y -> 89 | let d' = distance (hash y) x in 90 | if d' < !d then ( d := d'; n := y ) 91 | ) (StrMap.keys !_context.workers @ [!_context.myself_addr]); 92 | !n 93 | 94 | 95 | let nearest_exclude x l = 96 | let addrs = StrMap.keys !_context.workers @ [!_context.myself_addr] 97 | |> List.filter (fun x -> not (List.mem x l)) 98 | in 99 | let d = ref max_int in 100 | let n = ref "" in 101 | List.iteri (fun _i y -> 102 | let d' = distance (hash y) x in 103 | if d' < !d then ( d := d'; n := y ) 104 | ) addrs; 105 | !n 106 | 107 | 108 | let forward nxt typ msg = 109 | let s = StrMap.find nxt !_context.workers in 110 | Actor_utils.send ~bar:!_context.step s typ msg 111 | 112 | 113 | let init_table addrs = 114 | (* contact initial random peers *) 115 | Array.iter (fun x -> 116 | let s = connect x in 117 | let _ = add x s in 118 | Actor_utils.send s P2P_Ping [|!_context.myself_addr|]; 119 | ) addrs; 120 | (* contact nodes of fixed distance *) 121 | let myid = hash !_context.myself_addr in 122 | let _ = Array.init 30 (fun i -> 123 | let d = 2. ** (float_of_int i) |> int_of_float in 124 | let a = (myid + d) mod _space in 125 | let n = nearest_exclude a [!_context.myself_addr] in 126 | if String.length n <> 0 then 127 | let s = Marshal.to_string a [] in 128 | forward n P2P_Join [|!_context.myself_addr; s|] 129 | ) in () 130 | 131 | 132 | end 133 | 134 | 135 | let _get k = 136 | let k' = Obj.repr k in 137 | let v, t = Hashtbl.find _param k' in 138 | Obj.obj v, t 139 | 140 | 141 | let _set k v t = 142 | let k' = Obj.repr k in 143 | let v' = Obj.repr v in 144 | match Hashtbl.mem _param k' with 145 | | true -> Hashtbl.replace _param k' (v',t) 146 | | false -> Hashtbl.add _param k' (v',t) 147 | 148 | 149 | let _allocate_params x y = 150 | let x = Route.hash x in 151 | let y = Route.hash y in 152 | let l = ref [] in 153 | Hashtbl.iter (fun k v -> 154 | let h = Obj.obj k |> Route.hash in 155 | if (Route.distance y h) < (Route.distance x h) then l := !l @ [(k,v)] 156 | ) _param; !l 157 | 158 | 159 | let _shall_deliver_pull () = 160 | let ready = ref true in 161 | Hashtbl.iter (fun _k v -> 162 | match v with Some _ -> () | None -> ready := false 163 | ) _plbuf; 164 | if !ready = true then ( 165 | let s = Hashtbl.fold (fun _ v l -> 166 | match v with 167 | | Some v -> let k,v,_t = Obj.obj v in l @ [(k,v)] 168 | | None -> l 169 | ) _plbuf [] 170 | in 171 | let s = Marshal.to_string s [] in 172 | Actor_utils.send !_context.master_sock OK [|s|]; 173 | Hashtbl.reset _plbuf 174 | ) 175 | 176 | 177 | let _barrier_control barrier pull = 178 | let updates = List.map Obj.obj !_context.mpbuf in 179 | if barrier _context = true then ( 180 | pull _context updates |> List.iter (fun (k,v,t) -> _set k v t); 181 | !_context.mpbuf <- []; 182 | if !_context.block = true then ( 183 | Actor_utils.send ~bar:!_context.step !_context.master_sock OK [||]; 184 | !_context.block <- false 185 | ) 186 | ) 187 | 188 | 189 | let _update_step_buf addr step = 190 | if Hashtbl.mem !_context.spbuf addr = true then ( 191 | let step = max step (Hashtbl.find !_context.spbuf addr) in 192 | Hashtbl.replace !_context.spbuf addr step 193 | ) 194 | else Hashtbl.add !_context.spbuf addr step 195 | 196 | 197 | let _notify_peers_step () = 198 | List.iter (fun k -> 199 | Route.forward k P2P_Ping [|!_context.myself_addr|] 200 | ) (StrMap.keys !_context.workers) 201 | 202 | 203 | let _process_timeout () = 204 | _notify_peers_step (); 205 | Owl_log.debug "%s: timeout" !_context.myself_addr 206 | 207 | 208 | let service_loop () = 209 | Owl_log.debug "%s: p2p server" !_context.myself_addr; 210 | let barrier : p2p_barrier_typ = Marshal.from_string !_barrier 0 in 211 | let pull : ('a, 'b) p2p_pull_typ = Marshal.from_string !_pull 0 in 212 | (* loop to process messages *) 213 | ZMQ.Socket.set_receive_timeout !_context.myself_sock (1 * 1000); 214 | try while true do 215 | (* first, wait and process arriving message *) 216 | try let i, m = Actor_utils.recv !_context.myself_sock in ( 217 | match m.typ with 218 | | P2P_Connect -> ( 219 | Owl_log.debug "%s: p2p_connect %s" !_context.myself_addr m.par.(0); 220 | let addr = m.par.(0) in 221 | !_context.master_addr <- addr; 222 | !_context.master_sock <- Route.connect addr 223 | ) 224 | | P2P_Ping -> ( 225 | Owl_log.debug "%s: p2p_ping %s" !_context.myself_addr m.par.(0); 226 | let addr = m.par.(0) in 227 | if Route.exists addr = false then Route.(connect addr |> add addr) 228 | ) 229 | | P2P_Join -> ( 230 | Owl_log.debug "%s: p2p_join %s" !_context.myself_addr m.par.(0); 231 | let src = m.par.(0) in 232 | let dst = Marshal.from_string m.par.(1) 0 in 233 | let next = Route.nearest_exclude dst [src] in 234 | if next = !_context.myself_addr then ( 235 | if Route.exists src = false then ( 236 | let s = Route.connect src in 237 | let _ = Route.add src s in 238 | Actor_utils.send s P2P_Ping [|!_context.myself_addr|] 239 | ); 240 | (* oh, hello neighbour, maybe take my model *) 241 | if Route.hash src = dst - 1 then ( 242 | let next = Route.furthest_exclude dst [src; !_context.myself_addr] in 243 | if String.length next <> 0 then 244 | Route.forward next P2P_Ping [|src|]; 245 | let h = _allocate_params !_context.myself_addr src in 246 | let s = Marshal.to_string h [] in 247 | Owl_log.debug "params: %s ===> %s size:%i" !_context.myself_addr src (String.length s); 248 | Route.forward src P2P_Copy [|s|] 249 | ) 250 | ); 251 | if next <> !_context.myself_addr && String.length next <> 0 then 252 | Route.forward next P2P_Join m.par; 253 | ) 254 | | P2P_Copy -> ( 255 | Owl_log.debug "%s: p2p_copy" !_context.myself_addr; 256 | let h = Marshal.from_string m.par.(0) 0 in 257 | List.iter (fun (k,v) -> 258 | match Hashtbl.mem _param k with 259 | | true -> Hashtbl.replace _param k v 260 | | false -> Hashtbl.add _param k v 261 | ) h 262 | ) 263 | | P2P_Get -> ( 264 | Owl_log.debug "%s: p2p_get" !_context.myself_addr; 265 | let k = Marshal.from_string m.par.(0) 0 in 266 | let next = Route.(hash k |> nearest) in 267 | match next = !_context.myself_addr with 268 | | true -> ( 269 | (* FIXME: what if i cannot find the k *) 270 | let v, t = _get k in 271 | let s = Marshal.to_string (k, v, t) [] in 272 | Actor_utils.send !_context.master_sock OK [|s; next|] 273 | ) 274 | | false -> Route.forward next P2P_Get_Q m.par 275 | ) 276 | | P2P_Get_Q -> ( 277 | Owl_log.debug "%s: p2p_get_q" !_context.myself_addr; 278 | let k = Marshal.from_string m.par.(0) 0 in 279 | let next = Route.(hash k |> nearest) in 280 | match next = !_context.myself_addr with 281 | | true -> ( 282 | let v, t = _get k in 283 | let s = Marshal.to_string (k, v, t) [] in 284 | let addr = m.par.(1) in 285 | let next = Route.(hash addr |> nearest) in 286 | Route.forward next P2P_Get_R [|s; addr|] 287 | ) 288 | | false -> Route.forward next P2P_Get_Q m.par 289 | ) 290 | | P2P_Get_R -> ( 291 | Owl_log.debug "%s: p2p_get_r" !_context.myself_addr; 292 | let addr = m.par.(1) in 293 | let next = Route.(hash addr |> nearest) in 294 | match next = !_context.myself_addr with 295 | | true -> Actor_utils.send !_context.master_sock OK m.par 296 | | false -> Route.forward next P2P_Get_R m.par 297 | ) 298 | | P2P_Set -> ( 299 | Owl_log.debug "%s: p2p_get" !_context.myself_addr; 300 | let k, v, t = Marshal.from_string m.par.(0) 0 in 301 | (* check whether this is from the local client *) 302 | let t = if t < 0 then ( 303 | let s = Marshal.to_string (k, v, !_context.step) [] in 304 | m.par <- [|s|]; !_context.step 305 | ) else t 306 | in 307 | let next = Route.(hash k |> nearest) in 308 | match next = !_context.myself_addr with 309 | | true -> !_context.mpbuf <- !_context.mpbuf @ [Obj.repr (k, v, t)] 310 | | false -> Route.forward next P2P_Set m.par 311 | ) 312 | | P2P_Push -> ( 313 | Owl_log.debug "%s: p2p_push" !_context.myself_addr; 314 | Marshal.from_string m.par.(0) 0 315 | |> List.iter (fun (k,v) -> 316 | let next = Route.(hash k |> nearest) in 317 | match next = !_context.myself_addr with 318 | | true -> !_context.mpbuf <- !_context.mpbuf @ [Obj.repr (k, v, !_context.step)] 319 | | false -> ( 320 | let s = Marshal.to_string (k, v, !_context.step) [] in 321 | Route.forward next P2P_Set [|s|] 322 | ) 323 | ) 324 | ) 325 | | P2P_Pull -> ( 326 | Owl_log.debug "%s: p2p_pull" !_context.myself_addr; 327 | Marshal.from_string m.par.(0) 0 328 | |> List.iter (fun k -> 329 | let next = Route.(hash k |> nearest) in 330 | match next = !_context.myself_addr with 331 | | true -> ( 332 | let v, t = _get k in 333 | Hashtbl.add _plbuf (Obj.repr k) (Some (Obj.repr (k,v,t))) 334 | ) 335 | | false -> ( 336 | let y = Marshal.to_string k [] in 337 | let s = [|y; !_context.myself_addr|] in 338 | Route.forward next P2P_Pull_Q s; 339 | Hashtbl.add _plbuf (Obj.repr k) None 340 | ) 341 | ); 342 | _shall_deliver_pull () 343 | ) 344 | | P2P_Pull_Q -> ( 345 | Owl_log.debug "%s: p2p_pull_q %s" !_context.myself_addr m.par.(1); 346 | let k = Marshal.from_string m.par.(0) 0 in 347 | let next = Route.(hash k |> nearest) in 348 | match next = !_context.myself_addr with 349 | | true -> ( 350 | let v, t = _get k in 351 | let s = Marshal.to_string (k, v, t) [] in 352 | let addr = m.par.(1) in 353 | let next = Route.(hash addr |> nearest) in 354 | Route.forward next P2P_Pull_R [|s; addr|] 355 | ) 356 | | false -> Route.forward next P2P_Pull_Q m.par 357 | ) 358 | | P2P_Pull_R -> ( 359 | Owl_log.debug "%s: p2p_pull_r %s" !_context.myself_addr m.par.(1); 360 | let addr = m.par.(1) in 361 | let next = Route.(hash addr |> nearest) in 362 | match next = !_context.myself_addr with 363 | | true -> ( 364 | let k, v, t = Marshal.from_string m.par.(0) 0 in 365 | Hashtbl.replace _plbuf (Obj.repr k) (Some (Obj.repr (k,v,t))); 366 | _shall_deliver_pull () 367 | ) 368 | | false -> Route.forward next P2P_Pull_R m.par 369 | ) 370 | | P2P_Bar -> ( 371 | Owl_log.debug "%s: p2p_bar" !_context.myself_addr; 372 | !_context.block <- true; 373 | !_context.step <- !_context.step + 1; 374 | _notify_peers_step (); 375 | ) 376 | | _ -> Owl_log.error "unknown mssage type" 377 | ); 378 | (* second, update the piggybacked step *) 379 | if i <> !_context.master_addr then _update_step_buf i m.bar; 380 | (* third, check the barrier control *) 381 | _barrier_control barrier pull; 382 | (* fourth, in case the process hangs *) 383 | with Unix.Unix_error (_,_,_) -> _process_timeout () 384 | done with Failure e -> ( 385 | Owl_log.warn "%s" e; 386 | ZMQ.Socket.close !_context.myself_sock ) 387 | 388 | 389 | let init m context = 390 | _context := context; 391 | (* contact allocated peers to join the swarm *) 392 | Marshal.from_string m.par.(0) 0 |> Route.init_table; 393 | (* enter into server service loop *) 394 | service_loop () 395 | -------------------------------------------------------------------------------- /lib/actor_service.ml: -------------------------------------------------------------------------------- 1 | (* 2 | * Actor - Parallel & Distributed Engine of Owl System 3 | * Copyright (c) 2016-2018 Liang Wang 4 | *) 5 | 6 | (* Service: defines basic functionality of services *) 7 | 8 | open Actor_types 9 | 10 | 11 | let _services = ref StrMap.empty 12 | 13 | 14 | let mem id = StrMap.mem id !_services 15 | 16 | 17 | let add id master = 18 | let s = { id = id; master = master; worker = [||] } in 19 | _services := StrMap.add id s !_services 20 | 21 | 22 | let add_worker id wid = 23 | let service = StrMap.find id !_services in 24 | let workers = Array.append service.worker [| wid |] in 25 | service.worker <- workers 26 | 27 | 28 | let find id = StrMap.find id !_services 29 | 30 | 31 | let choose_workers id n = 32 | let s = StrMap.find id !_services in 33 | match Array.length s.worker > n with 34 | | true -> Owl.Stats.choose s.worker n 35 | | false -> s.worker 36 | -------------------------------------------------------------------------------- /lib/actor_storage.ml: -------------------------------------------------------------------------------- 1 | (* 2 | * Actor - Parallel & Distributed Engine of Owl System 3 | * Copyright (c) 2016-2018 Liang Wang 4 | *) 5 | 6 | (* Storage module: provide a basic persistent storage service *) 7 | 8 | let unix_load x = 9 | let l = Unix.((stat x).st_size) in 10 | let b = Bytes.create l in 11 | let f = Unix.(openfile x [O_RDONLY] 0o644) in 12 | let _ = Unix.read f b 0 l in b 13 | 14 | 15 | let unix_save x b = 16 | let f = Unix.(openfile x [O_WRONLY; O_CREAT] 0o644) in 17 | let l = Bytes.length b in 18 | Unix.write f b 0 l 19 | -------------------------------------------------------------------------------- /lib/actor_types.ml: -------------------------------------------------------------------------------- 1 | (* 2 | * Actor - Parallel & Distributed Engine of Owl System 3 | * Copyright (c) 2016-2018 Liang Wang 4 | *) 5 | 6 | (* Types: includes the types shared by different modules. *) 7 | 8 | 9 | module StrMap = struct 10 | include Map.Make (String) 11 | let keys x = List.map fst (bindings x) 12 | let values x = List.map snd (bindings x) 13 | end 14 | 15 | 16 | type color = Red | Green | Blue 17 | 18 | 19 | type message_type = 20 | (* General messge types *) 21 | | OK | Fail | Heartbeat | User_Reg | Job_Reg | Job_Master | Job_Worker | Job_Create 22 | (* Data parallel: Mapre *) 23 | | MapTask | MapPartTask | FilterTask | ReduceByKeyTask | ShuffleTask | UnionTask 24 | | JoinTask | FlattenTask | ApplyTask | NopTask 25 | | Pipeline | Collect | Count | Broadcast | Fold | Reduce | Terminate | Load | Save 26 | (* Model Parallel: Param *) 27 | | PS_Get | PS_Set | PS_Schedule | PS_Push 28 | (* P2P Parallel: Peer *) 29 | | P2P_Reg | P2P_Connect | P2P_Ping | P2P_Join | P2P_Forward | P2P_Set | P2P_Get 30 | | P2P_Get_Q | P2P_Get_R | P2P_Copy | P2P_Push | P2P_Pull | P2P_Pull_Q | P2P_Pull_R 31 | | P2P_Bar 32 | 33 | 34 | type message_rec = { 35 | mutable bar : int; 36 | mutable typ : message_type; 37 | mutable par : string array; 38 | } 39 | 40 | 41 | type mapre_context = { 42 | mutable ztx : ZMQ.Context.t; (* zmq context for communication *) 43 | mutable job_id : string; (* job id or swarm id, depends on paradigm *) 44 | mutable master_addr : string; (* different meaning in different paradigm *) 45 | mutable myself_addr : string; (* communication address of current process *) 46 | mutable master_sock : [`Dealer] ZMQ.Socket.t; (* socket of master_addr *) 47 | mutable myself_sock : [`Router] ZMQ.Socket.t; (* socket of myself_addr *) 48 | mutable workers : [`Dealer] ZMQ.Socket.t StrMap.t; (* socket of workers or peers *) 49 | mutable step : int; (* local step for barrier control *) 50 | mutable msbuf : (int, string * message_rec) Hashtbl.t; (* buffer of un-ordered messages *) 51 | } 52 | 53 | 54 | type param_context = { 55 | mutable ztx : ZMQ.Context.t; (* zmq context for communication *) 56 | mutable job_id : string; (* job id or swarm id, depends on paradigm *) 57 | mutable master_addr : string; (* different meaning in different paradigm *) 58 | mutable myself_addr : string; (* communication address of current process *) 59 | mutable master_sock : [`Dealer] ZMQ.Socket.t; (* socket of master_addr *) 60 | mutable myself_sock : [`Router] ZMQ.Socket.t; (* socket of myself_addr *) 61 | mutable workers : [`Dealer] ZMQ.Socket.t StrMap.t; (* socket of workers or peers *) 62 | mutable step : int; (* local step for barrier control *) 63 | mutable stale : int; (* staleness variable for barrier control *) 64 | mutable worker_busy : (string, int) Hashtbl.t; (* lookup table of a worker busy or not *) 65 | mutable worker_step : (string, int) Hashtbl.t; (* lookup table of a worker's step *) 66 | mutable step_worker : (int, string) Hashtbl.t; (* lookup table of workers at a specific step *) 67 | } 68 | 69 | 70 | type peer_context = { 71 | mutable ztx : ZMQ.Context.t; (* zmq context for communication *) 72 | mutable job_id : string; (* job id or swarm id, depends on paradigm *) 73 | mutable master_addr : string; (* different meaning in different paradigm *) 74 | mutable myself_addr : string; (* communication address of current process *) 75 | mutable master_sock : [`Dealer] ZMQ.Socket.t; (* socket of master_addr *) 76 | mutable myself_sock : [`Router] ZMQ.Socket.t; (* socket of myself_addr *) 77 | mutable workers : [`Dealer] ZMQ.Socket.t StrMap.t; (* socket of workers or peers *) 78 | mutable step : int; (* local step for barrier control *) 79 | mutable block : bool; (* is client blocked at barrier *) 80 | mutable mpbuf : Obj.t list; (* buffer of model parameter updates *) 81 | mutable spbuf : (string, int) Hashtbl.t; (* buffer of the step of connected peers, piggybacked in m.bar *) 82 | } 83 | 84 | 85 | type actor_rec = { 86 | id : string; 87 | addr : string; 88 | last_seen : float; 89 | } 90 | 91 | 92 | type data_rec = { 93 | id : string; 94 | owner : string; 95 | } 96 | 97 | 98 | type service_rec = { 99 | id : string; 100 | master : string; 101 | mutable worker : string array; 102 | } 103 | 104 | 105 | (** types of user-defined functions in model parallel module *) 106 | 107 | type ('a, 'b, 'c) ps_schedule_typ = 'a list -> ('a * ('b * 'c) list) list 108 | 109 | type ('a, 'b, 'c) ps_pull_typ = ('a * 'b) list -> ('a * 'c) list 110 | 111 | type ('a, 'b, 'c) ps_push_typ = 'a -> ('b * 'c) list -> ('b * 'c) list 112 | 113 | type ps_barrier_typ = param_context ref -> int * (string list) 114 | 115 | type ps_stop_typ = param_context ref -> bool 116 | 117 | 118 | (** types of user-defined functions in p2p parallel module *) 119 | 120 | type 'a p2p_schedule_typ = peer_context ref -> 'a list 121 | 122 | type ('a, 'b) p2p_pull_typ = peer_context ref -> ('a * 'b * int) list -> ('a * 'b * int) list 123 | 124 | type ('a, 'b) p2p_push_typ = peer_context ref -> ('a * 'b) list -> ('a * 'b) list 125 | 126 | type p2p_barrier_typ = peer_context ref -> bool 127 | 128 | type p2p_stop_typ = peer_context ref -> bool 129 | 130 | 131 | (** two functions to translate between message rec and string *) 132 | 133 | let to_msg b t p = 134 | let m = { bar = b; typ = t; par = p } in 135 | Marshal.to_string m [ ] 136 | 137 | 138 | let of_msg s = 139 | let m : message_rec = Marshal.from_string s 0 in m;; 140 | 141 | 142 | (* initialise some states *) 143 | 144 | Random.self_init ();; 145 | -------------------------------------------------------------------------------- /lib/actor_utils.ml: -------------------------------------------------------------------------------- 1 | (* 2 | * Actor - Parallel & Distributed Engine of Owl System 3 | * Copyright (c) 2016-2018 Liang Wang 4 | *) 5 | 6 | (** Some shared helper functions *) 7 | 8 | open Actor_types 9 | 10 | 11 | let recv s = 12 | let m = ZMQ.Socket.recv_all ~block:true s in 13 | (List.nth m 0, List.nth m 1 |> of_msg) 14 | 15 | 16 | let send ?(bar=0) v t s = 17 | try ZMQ.Socket.send ~block:false v (to_msg bar t s) 18 | with _exn -> let hwm = ZMQ.Socket.get_send_high_water_mark v in 19 | Owl_log.error "fail to send bar:%i hwm:%i" bar hwm 20 | 21 | 22 | let rec _bind_available_addr addr sock ztx = 23 | addr := "tcp://127.0.0.1:" ^ (string_of_int (Random.int 10000 + 50000)); 24 | try ZMQ.Socket.bind sock !addr 25 | with _exn -> _bind_available_addr addr sock ztx 26 | 27 | 28 | let bind_available_addr ztx = 29 | let router : [`Router] ZMQ.Socket.t = ZMQ.Socket.create ztx ZMQ.Socket.router in 30 | let addr = ref "" in _bind_available_addr addr router ztx; 31 | ZMQ.Socket.set_receive_high_water_mark router Actor_config.high_warter_mark; 32 | !addr, router 33 | 34 | 35 | (* the following 3 functions are for shuffle operations *) 36 | 37 | let _group_by_key x = 38 | let h = Hashtbl.create 1_024 in 39 | List.iter (fun (k,v) -> 40 | match Hashtbl.mem h k with 41 | | true -> Hashtbl.replace h k ((Hashtbl.find h k) @ [v]) 42 | | false -> Hashtbl.add h k [v] 43 | ) x; 44 | Hashtbl.fold (fun k v l -> (k,v) :: l) h [] 45 | 46 | 47 | let group_by_key x = (* FIXME: stack overflow if there too many values for a key *) 48 | let h, g = Hashtbl.(create 1_024, create 1_024) in 49 | List.iter (fun (k,v) -> Hashtbl.(add h k v; if not (mem g k) then add g k None)) x; 50 | Hashtbl.fold (fun k _ l -> (k,Hashtbl.find_all h k) :: l) g [] 51 | 52 | 53 | let flatten_kvg x = 54 | try List.map (fun (k,l) -> List.map (fun v -> (k,v)) l) x |> List.flatten 55 | with _exn -> print_endline "Error: flatten_kvg"; [] 56 | 57 | 58 | let choose_load x n i = List.filter (fun (k,_l) -> (Hashtbl.hash k mod n) = i) x 59 | 60 | 61 | (* generate a log file name from address *) 62 | let addr_to_log x = 63 | let path = Str.(split (regexp "://")) x in 64 | List.nth path 1 |> Str.(global_replace (regexp "[:.]") "_") 65 | 66 | 67 | let empty_mapre_context () = 68 | let ztx = ZMQ.Context.create () in 69 | { 70 | ztx = ztx; 71 | job_id = ""; 72 | master_addr = ""; 73 | myself_addr = ""; 74 | master_sock = ZMQ.Socket.(create ztx dealer); 75 | myself_sock = ZMQ.Socket.(create ztx router); 76 | workers = StrMap.empty; 77 | step = 0; 78 | msbuf = Hashtbl.create 256; 79 | } 80 | 81 | 82 | let empty_param_context () = 83 | let ztx = ZMQ.Context.create () in 84 | { 85 | ztx = ztx; 86 | job_id = ""; 87 | master_addr = ""; 88 | myself_addr = ""; 89 | master_sock = ZMQ.Socket.(create ztx dealer); 90 | myself_sock = ZMQ.Socket.(create ztx router); 91 | workers = StrMap.empty; 92 | step = 0; 93 | stale = 1; 94 | worker_busy = Hashtbl.create 1_000; 95 | worker_step = Hashtbl.create 1_000; 96 | step_worker = Hashtbl.create 1_000; 97 | } 98 | 99 | 100 | let empty_peer_context () = 101 | let ztx = ZMQ.Context.create () in 102 | { 103 | ztx = ztx; 104 | job_id = ""; 105 | master_addr = ""; 106 | myself_addr = ""; 107 | master_sock = ZMQ.Socket.(create ztx dealer); 108 | myself_sock = ZMQ.Socket.(create ztx router); 109 | workers = StrMap.empty; 110 | step = 0; 111 | block = false; 112 | mpbuf = []; 113 | spbuf = Hashtbl.create 32; 114 | } 115 | -------------------------------------------------------------------------------- /lib/jbuild: -------------------------------------------------------------------------------- 1 | (jbuild_version 1) 2 | 3 | (library 4 | ((public_name actor) 5 | (name actor) 6 | (modules 7 | (:standard \ 8 | (actor_peer_sgd0 9 | actor_peer_sgd1 10 | actor_param_sgd 11 | ))) 12 | (flags ( 13 | ; turn off some warnings as 4.06.0 is pernickety and errors out 14 | :standard -w "-50" -safe-string) 15 | ) 16 | (wrapped false) 17 | (libraries ( 18 | zmq 19 | ocamlgraph 20 | owl 21 | ) 22 | ))) 23 | -------------------------------------------------------------------------------- /test/jbuild: -------------------------------------------------------------------------------- 1 | (jbuild_version 1) 2 | 3 | (executables 4 | ((public_names (test_wordcount test_kmeans test_owl_parallel)) 5 | (names (test_wordcount test_kmeans test_owl_parallel)) 6 | (libraries ( 7 | actor 8 | owl 9 | )))) 10 | -------------------------------------------------------------------------------- /test/test_context.ml: -------------------------------------------------------------------------------- 1 | (** [] 2 | test the context module 3 | *) 4 | 5 | let print_float_list x = 6 | List.iter (fun y -> Printf.printf "%.2f\t" y) x; 7 | print_endline "" 8 | 9 | let print_kv_list x = 10 | List.iter (fun l -> 11 | if (List.length l) > 0 then 12 | ( List.iter (fun (k,v) -> Printf.printf "(%c,%.2f)\t" k v) l; print_endline "" ) 13 | ) x 14 | 15 | let print_join_list x = 16 | List.iter (fun l -> 17 | if (List.length l) > 0 then 18 | ( List.iter (fun (k,l1) -> 19 | Printf.printf "%c : " k; 20 | List.iter (fun v -> Printf.printf "%.2f; " v) l1 ) l; 21 | print_endline "" ) 22 | ) x 23 | 24 | let test () = 25 | Actor_mapre.init Sys.argv.(1) "tcp://localhost:5555"; 26 | (* Test map *) 27 | let x1 = Actor_mapre.map (fun x -> x *. 2.) "default" in 28 | (* Test broadcast *) 29 | let y = Actor_mapre.broadcast 3. in 30 | let x2 = Actor_mapre.map (fun x -> x +. (Actor_mapre.get_value y)) x1 in 31 | (* Test union *) 32 | let x3 = Actor_mapre.union x1 x2 in 33 | (* Test filter *) 34 | let x4 = Actor_mapre.filter ((>) 10.) x3 in 35 | (* Test shuffle & reduce_by_key *) 36 | let x5 = Actor_mapre.map (fun x -> if x > 10. then ('a',x) else ('b',x)) x3 in 37 | let x7 = Actor_mapre.reduce_by_key (max) x5 in 38 | let _ = (* print_kv_list *) (Actor_mapre.collect x5) in 39 | (* collect data *) 40 | let _ = (* List.iter (fun x -> print_float_list x) *) (Actor_mapre.collect x3) in 41 | Printf.printf "num of x4 is %s\n" (Actor_mapre.count x4 |> string_of_int); 42 | Printf.printf "sum of x3 is %.2f\n" (Actor_mapre.fold (+.) 0. x3); 43 | Printf.printf "max of x3 is %.2f\n" (Actor_mapre.fold max 0. x3); 44 | print_kv_list (Actor_mapre.collect x7); 45 | (* test join *) 46 | (* let x8 = Actor_mapre.join x5 x5 in 47 | print_join_list (Actor_mapre.collect x8); *) 48 | (* Terminate *) 49 | Actor_mapre.terminate () 50 | 51 | let _ = test () 52 | -------------------------------------------------------------------------------- /test/test_coordinate.ml: -------------------------------------------------------------------------------- 1 | (** [ Test coordinate descent ] *) 2 | 3 | module PS = Actor_param 4 | 5 | let param = Array.(init 1000 (fun x -> x) |> to_list) 6 | 7 | let pivot = ref 0 8 | 9 | let schedule workers = 10 | Owl_log.debug "scheduling ..."; 11 | let tasks = List.map (fun x -> 12 | let k, v = !pivot, 0.5 in 13 | pivot := !pivot + 1; 14 | (x, [(k,v)]) 15 | ) workers in 16 | Owl_log.debug "scheduling done ..."; 17 | tasks 18 | 19 | let push = None 20 | 21 | let pull = None 22 | 23 | let test_coordinate_descent () = 24 | PS.register_schedule schedule; 25 | PS.start Sys.argv.(1) Actor_config.manager_addr; 26 | Owl_log.info "do some work at master node" 27 | 28 | let _ = test_coordinate_descent () 29 | -------------------------------------------------------------------------------- /test/test_irmin.ml: -------------------------------------------------------------------------------- 1 | (** [ Test Irmin ] *) 2 | 3 | open Lwt 4 | 5 | module Store = Irmin_mem.AO (Irmin.Hash.SHA1) (Irmin.Contents.String) 6 | 7 | let config = Irmin_mem.config () 8 | 9 | let test () = 10 | let s = Store.create config 11 | >>= fun s -> Store.add s "hello" 12 | >>= fun k -> Store.read s k 13 | >>= function 14 | | Some v -> print_endline v; Lwt.return_unit 15 | | None -> print_endline "None"; Lwt.return_unit 16 | in s 17 | 18 | let _ = Lwt_main.run (test ()) 19 | -------------------------------------------------------------------------------- /test/test_kmeans.ml: -------------------------------------------------------------------------------- 1 | (** [ Naive K-means implementation ] 2 | *) 3 | 4 | module Ctx = Actor.Mapre 5 | 6 | let print_points = List.iter (fun x -> Printf.printf "(%.2f,%.2f)\n" (fst x) (snd x)) 7 | let distance x y = (((fst x) -. (fst y)) ** 2.) +. (((snd x) -. (snd y)) ** 2.) 8 | let add_2pts x y = ( ((fst x)+.(fst y)), ((snd x)+.(snd y)) ) 9 | 10 | let format_filter_data fname = fname 11 | |> Ctx.flatmap Str.(split (regexp "[\r\n]")) 12 | |> Ctx.map Str.(split (regexp "[,]")) 13 | |> Ctx.map (fun x -> (float_of_string (List.nth x 0),float_of_string (List.nth x 1))) 14 | |> Ctx.filter (fun _ -> Random.float 1. < 0.1) 15 | 16 | let kmeans x = 17 | let centers = ref [(1.,1.); (2.,2.); (3.,3.); (4.,4.)] in 18 | for _i = 0 to 50 do 19 | let bc = Ctx.broadcast !centers in 20 | let y = Ctx.map (fun x -> 21 | let k = ref (-1, max_float) in 22 | Ctx.get_value bc |> List.iteri (fun i y -> 23 | let d = distance x y in 24 | if d < (snd !k) then k := (i,d) ); 25 | (fst !k, (x, 1.)) ) x in 26 | let y = Ctx.reduce_by_key (fun x y -> (add_2pts (fst x) (fst y), (snd x)+.(snd y)) ) y in 27 | centers := Ctx.collect y 28 | |> List.flatten 29 | |> List.map (fun (_k,v) -> let p, c = v in ((fst p)/.c,(snd p)/.c) ) 30 | done; 31 | !centers 32 | 33 | let _ = 34 | Ctx.init Sys.argv.(1) "tcp://localhost:5555"; 35 | Ctx.load "unix://data/kmeans.data" |> format_filter_data |> kmeans |> print_points; 36 | Ctx.terminate () 37 | -------------------------------------------------------------------------------- /test/test_owl_parallel.ml: -------------------------------------------------------------------------------- 1 | (** [ Test Parallel Module in Owl ] *) 2 | 3 | module Ctx = Actor.Mapre 4 | 5 | let test_naive () = 6 | Ctx.init Sys.argv.(1) "tcp://localhost:5555"; 7 | Ctx.load "default" |> Ctx.map (fun _ -> print_endline "hello") |> ignore; 8 | Ctx.terminate () 9 | 10 | (** [ Test parameter server ] *) 11 | 12 | (* 13 | module PS = Actor_param 14 | 15 | let retrieve_model () = 16 | let open Owl.Neural.S in 17 | let open Graph in 18 | (* model has been initialised *) 19 | try PS.get "model" 20 | (* model does not exist, init *) 21 | with Not_found -> ( 22 | Owl_log.warn "model does not exists, init now ..."; 23 | let nn = 24 | input [|28;28;1|] 25 | |> conv2d [|5;5;1;32|] [|1;1|] ~act_typ:Activation.Relu 26 | |> max_pool2d [|2;2|] [|2;2|] 27 | |> conv2d [|5;5;32;64|] [|1;1|] ~act_typ:Activation.Relu 28 | |> max_pool2d [|2;2|] [|2;2|] 29 | |> dropout 0.1 30 | |> fully_connected 1024 ~act_typ:Activation.Relu 31 | |> linear 10 ~act_typ:Activation.Softmax 32 | in 33 | PS.set "model" nn; 34 | PS.get "model" 35 | ) 36 | 37 | 38 | let schedule workers = 39 | let model = retrieve_model () |> fst in 40 | Owl.Neural.S.Graph.print model; flush_all (); 41 | let tasks = List.map (fun x -> 42 | (x, [("model", model)]) 43 | ) workers in tasks 44 | 45 | 46 | let push id vars = 47 | let updates = List.map (fun (k, model) -> 48 | (* 49 | let sleep_t = Owl.Stats.Rnd.uniform_int ~a:1 ~b:10 () in 50 | Owl_log.info "working %is" sleep_t; 51 | Owl_neural.Feedforward.print model; flush_all (); 52 | Unix.sleep sleep_t; 53 | *) 54 | let open Owl.Neural.S in 55 | 56 | let x, _, y = Owl.Dataset.load_cifar_train_data 1 in 57 | 58 | let params = Params.config 59 | ~batch:(Batch.Sample 100) ~learning_rate:(Learning_Rate.Adagrad 0.005) 0.01 in 60 | Owl.Neural.S.Graph.train_cnn ~params model x y |> ignore; 61 | 62 | (k, model) ) vars in 63 | updates 64 | 65 | 66 | let pull vars = 67 | List.map (fun (k,d) -> 68 | (* 69 | let v0, _ = PS.get k in 70 | let v1 = MX.(v0 - d) in 71 | *) 72 | let v1 = d in 73 | (k,v1) 74 | ) vars 75 | 76 | 77 | let test_param () = 78 | PS.register_schedule schedule; 79 | PS.register_push push; 80 | PS.register_barrier Actor_barrier.param_asp; 81 | 82 | PS.start Sys.argv.(1) Actor_config.manager_addr; 83 | Owl_log.info "do some work at master node" 84 | *) 85 | 86 | (* test parameter server engine *) 87 | module M2 = Owl_neural_parallel.Make (Owl.Neural.S.Graph) (Actor.Param) 88 | let test_neural_parallel () = 89 | let open Owl.Neural.S in 90 | let open Graph in 91 | let nn = 92 | input [|32;32;3|] 93 | |> normalisation ~decay:0.9 94 | |> conv2d [|3;3;3;32|] [|1;1|] ~act_typ:Activation.Relu 95 | |> conv2d [|3;3;32;32|] [|1;1|] ~act_typ:Activation.Relu ~padding:VALID 96 | |> max_pool2d [|2;2|] [|2;2|] ~padding:VALID 97 | |> dropout 0.1 98 | |> conv2d [|3;3;32;64|] [|1;1|] ~act_typ:Activation.Relu 99 | |> conv2d [|3;3;64;64|] [|1;1|] ~act_typ:Activation.Relu ~padding:VALID 100 | |> max_pool2d [|2;2|] [|2;2|] ~padding:VALID 101 | |> dropout 0.1 102 | |> fully_connected 512 ~act_typ:Activation.Relu 103 | |> linear 10 ~act_typ:Activation.(Softmax 1) 104 | |> get_network 105 | in 106 | 107 | let x, _, y = Owl.Dataset.load_cifar_train_data 1 in 108 | (* 109 | let params = Params.config 110 | ~batch:(Batch.Mini 100) ~learning_rate:(Learning_Rate.Adagrad 0.002) 0.05 in 111 | *) 112 | let chkpt state = 113 | if Checkpoint.(state.current_batch mod 1 = 0) then ( 114 | Checkpoint.(state.stop <- true); 115 | (* Log.info "sync model with server" *) 116 | ) 117 | in 118 | 119 | let params = Params.config 120 | ~batch:(Batch.Sample 100) ~learning_rate:(Learning_Rate.Adagrad 0.001) 121 | ~checkpoint:(Checkpoint.Custom chkpt) ~stopping:(Stopping.Const 1e-6) 10. 122 | in 123 | let url = Actor_config.manager_addr in 124 | let jid = Sys.argv.(1) in 125 | M2.train ~params nn x y jid url 126 | 127 | 128 | module M1 = Owl_parallel.Make_Distributed (Owl.Dense.Ndarray.D) (Actor.Mapre) 129 | let test_owl_distributed () = 130 | Ctx.init Sys.argv.(1) "tcp://localhost:5555"; 131 | (* some tests ... *) 132 | let x = M1.uniform [|2;3;4|] in 133 | Owl.Dense.Ndarray.D.print (M1.to_ndarray x); flush_all (); 134 | 135 | let y = M1.init [|2;3;4|] float_of_int in 136 | let _ = M1.set y [|1;2;3|] 0. in 137 | let y = M1.add x y in 138 | Owl.Dense.Ndarray.D.print (M1.to_ndarray y); flush_all (); 139 | 140 | let a = M1.get y [|1;2;2|] in 141 | Owl_log.info "get ===> %g" a; 142 | 143 | let x = M1.ones [|200;300;400|] in 144 | let x = M1.map (fun a -> a +. 1.) x in 145 | let a = M1.fold (+.) x 0. in 146 | let b = M1.sum x in 147 | Owl_log.info "fold vs. sum ===> %g, %g" a b; 148 | 149 | Owl_log.info "start retrieving big x"; 150 | let x = M1.to_ndarray x in 151 | Owl_log.info "finsh retrieving big x"; 152 | Owl_log.info "sum x = %g" (Owl.Arr.sum' x) 153 | 154 | 155 | let _ = test_neural_parallel () 156 | -------------------------------------------------------------------------------- /test/test_param_sgd.ml: -------------------------------------------------------------------------------- 1 | (** [ Test stochastic gradient descent ] *) 2 | 3 | open Owl 4 | module MX = Mat 5 | 6 | let data_x = MX.uniform 1000 3 7 | (*let data_y = let p = MX.of_array [|0.3;0.5;0.7|] 3 1 in MX.(data_x $@ p) 8 | let model = MX.of_array [|0.1;0.1;0.1|] 3 1*) 9 | let data_y = let p = MX.of_array [|0.3;0.5;0.7;0.4;0.9;0.2|] 3 2 in MX.(data_x *@ p) 10 | let model = MX.of_array [|0.1;0.1;0.1;0.1;0.1;0.1|] 3 2 11 | let gradfn = Owl_optimise.square_grad 12 | let lossfn = Owl_optimise.square_loss 13 | 14 | let _ = 15 | Actor_param_sgd.init data_x data_y model gradfn lossfn; 16 | Actor_param_sgd.start Sys.argv.(1) 17 | -------------------------------------------------------------------------------- /test/test_parameter.ml: -------------------------------------------------------------------------------- 1 | (** [ Test parameter server ] *) 2 | 3 | module PS = Actor_param 4 | 5 | let schedule workers = 6 | let tasks = List.map (fun x -> 7 | let k, v = Random.int 100, Random.int 1000 in (x, [(k,v)]) 8 | ) workers in tasks 9 | 10 | let push id vars = 11 | let updates = List.map (fun (k,v) -> 12 | Owl_log.info "working on %i" v; 13 | (k,v) ) vars in 14 | updates 15 | 16 | let test_context () = 17 | PS.register_schedule schedule; 18 | PS.register_push push; 19 | PS.start Sys.argv.(1) Actor_config.manager_addr; 20 | Owl_log.info "do some work at master node" 21 | 22 | let _ = test_context () 23 | -------------------------------------------------------------------------------- /test/test_peer.ml: -------------------------------------------------------------------------------- 1 | (** [ Test P2P parallel ] *) 2 | 3 | module P2P = Actor_peer 4 | 5 | let schedule id = 6 | (* Logger.debug "%s: scheduling ..." id; *) 7 | Unix.sleep 5; [] 8 | 9 | let test_context () = 10 | P2P.register_schedule schedule; 11 | P2P.start Sys.argv.(1) Actor_config.manager_addr 12 | 13 | let _ = test_context () 14 | -------------------------------------------------------------------------------- /test/test_peer_lda.ml: -------------------------------------------------------------------------------- 1 | (** [ Test LDA on P2P parallel ] *) 2 | 3 | open Owl 4 | open Actor_types 5 | 6 | (* load stopwords, load data, build dict, tokenisation *) 7 | let s = Dataset.load_stopwords () 8 | let x = Dataset.load_nips_train_data s 9 | let v, _ = Owl_nlp_utils.build_vocabulary x 10 | (* only choose 30% data to train 11 | let x = Stats.choose x (Array.length x / 3) *) 12 | let d = Owl_nlp_utils.tokenise_all v x 13 | let t = 100 14 | 15 | let _ = 16 | Owl_log.info "#doc:%i #top:%i #voc:%i" (Array.length d) t (Hashtbl.length v); 17 | Actor_peer_lda.init t v d; 18 | Actor_peer_lda.start Sys.argv.(1) 19 | -------------------------------------------------------------------------------- /test/test_peer_sgd.ml: -------------------------------------------------------------------------------- 1 | (** [ Test SGD on P2P parallel ] *) 2 | 3 | open Owl 4 | module MX = Mat 5 | 6 | let data_x = MX.uniform 1000 3 7 | let data_y = let p = MX.of_array [|0.3;0.5;0.7;0.4;0.9;0.2|] 3 2 in MX.(data_x *@ p) 8 | let model = MX.of_array [|0.1;0.1;0.1;0.1;0.1;0.1|] 3 2 9 | let gradfn = Owl_optimise.square_grad 10 | let lossfn = Owl_optimise.square_loss 11 | 12 | let _ = 13 | Actor_peer_sgd1.init data_x data_y model gradfn lossfn; 14 | Actor_peer_sgd1.start Sys.argv.(1) 15 | -------------------------------------------------------------------------------- /test/test_serialise.ml: -------------------------------------------------------------------------------- 1 | (** [ test_serialise ] 2 | test the serialisaton function 3 | *) 4 | 5 | let test () = print_endline "I am a test fun." 6 | 7 | let server () = 8 | let context = ZMQ.Context.create () in 9 | let responder = ZMQ.Socket.create context ZMQ.Socket.rep in 10 | ZMQ.Socket.bind responder "tcp://*:5555"; 11 | while true do 12 | let request = ZMQ.Socket.recv responder in 13 | Printf.printf "Received request: [%s]\n%!" request; 14 | let s = Marshal.to_string test [ Marshal.Closures ] in 15 | ZMQ.Socket.send responder s; 16 | done; 17 | ZMQ.Socket.close responder; 18 | ZMQ.Context.terminate context 19 | 20 | let client () = 21 | let context = ZMQ.Context.create () in 22 | print_endline "Connecting to hello world server..."; 23 | let requester = ZMQ.Socket.create context ZMQ.Socket.req in 24 | ZMQ.Socket.connect requester "tcp://localhost:5555"; 25 | for i = 1 to 5 do 26 | Printf.printf "Sending request %d...\n" i; 27 | ZMQ.Socket.send requester "Hello"; 28 | let s = ZMQ.Socket.recv requester in 29 | Printf.printf "call the serialized function %i ... \n" i; 30 | let f : unit -> unit = Marshal.from_string s 0 in 31 | f () 32 | done; 33 | ZMQ.Socket.close requester; 34 | ZMQ.Context.terminate context 35 | 36 | let _ = 37 | match Sys.argv.(1) with 38 | | "0" -> server () 39 | | "1" -> client () 40 | | _ -> print_endline "unknown" 41 | -------------------------------------------------------------------------------- /test/test_wordcount.ml: -------------------------------------------------------------------------------- 1 | (** [] 2 | test the context module 3 | *) 4 | 5 | module Ctx = Actor.Mapre 6 | 7 | let print_result x = List.iter (fun (k,v) -> Printf.printf "%s : %i\n" k v) x 8 | 9 | let stop_words = ["a";"are";"is";"in";"it";"that";"this";"and";"to";"of";"so"; 10 | "will";"can";"which";"for";"on";"in";"an";"with";"the";"-"] 11 | 12 | let wordcount () = 13 | Ctx.init Sys.argv.(1) "tcp://localhost:5555"; 14 | Ctx.load "unix://data/wordcount.data" 15 | |> Ctx.flatmap Str.(split (regexp "[ \t\n]")) 16 | |> Ctx.map String.lowercase_ascii 17 | |> Ctx.filter (fun x -> (String.length x) > 0) 18 | |> Ctx.filter (fun x -> not (List.mem x stop_words)) 19 | |> Ctx.map (fun k -> (k,1)) 20 | |> Ctx.reduce_by_key (+) 21 | |> Ctx.collect 22 | |> List.flatten |> print_result; 23 | Ctx.terminate () 24 | 25 | let _ = wordcount () 26 | --------------------------------------------------------------------------------