├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── README.md ├── demo ├── c++ │ ├── Makefile │ ├── README.md │ ├── echo.cpp │ ├── maelstrom.cpp │ └── maelstrom.h ├── clojure │ ├── echo.clj │ ├── flake_ids.clj │ ├── gcounter.clj │ ├── gossip.clj │ ├── gset.clj │ ├── kafka.clj │ ├── kafka_single_node.clj │ ├── multi_key_txn.clj │ ├── node.clj │ ├── single_key_txn.clj │ ├── txn_rw_register_hat.clj │ └── txn_rw_register_no_isolation.clj ├── go │ ├── README.md │ ├── cmd │ │ └── maelstrom-echo │ │ │ └── main.go │ ├── go.mod │ ├── kv.go │ ├── kv_test.go │ ├── node.go │ ├── node_test.go │ ├── rpc_error.go │ └── rpc_error_test.go ├── java │ ├── .gitignore │ ├── .vscode │ │ ├── launch.json │ │ ├── settings.json │ │ └── tasks.json │ ├── README.md │ ├── build │ ├── pom.xml │ ├── server │ └── src │ │ ├── main │ │ └── java │ │ │ └── maelstrom │ │ │ ├── Error.java │ │ │ ├── IJson.java │ │ │ ├── JsonUtil.java │ │ │ ├── Main.java │ │ │ ├── Message.java │ │ │ ├── Node1.java │ │ │ ├── Node2.java │ │ │ ├── Node3.java │ │ │ ├── broadcast │ │ │ └── BroadcastServer.java │ │ │ ├── echo │ │ │ └── EchoServer.java │ │ │ └── txnListAppend │ │ │ ├── AppendOp.java │ │ │ ├── IOp.java │ │ │ ├── KVStore.java │ │ │ ├── ReadOp.java │ │ │ ├── Root.java │ │ │ ├── State.java │ │ │ ├── Thunk.java │ │ │ ├── Txn.java │ │ │ └── TxnListAppendServer.java │ │ └── test │ │ └── java │ │ └── maelstrom │ │ └── AppTest.java ├── js │ ├── crdt_gset.js │ ├── crdt_pn_counter.js │ ├── echo.js │ ├── echo_minimal.js │ ├── gossip.js │ ├── multi_key_txn.js │ ├── node.js │ └── single_key_txn.js ├── python │ ├── README.md │ ├── broadcast.py │ ├── echo.py │ ├── maelstrom.py │ └── raft.py ├── ruby │ ├── broadcast.rb │ ├── crdt.rb │ ├── datomic_list_append.rb │ ├── echo.rb │ ├── echo_full.rb │ ├── errors.rb │ ├── g_set.rb │ ├── lin_kv_proxy.rb │ ├── node.rb │ ├── pn_counter.rb │ ├── promise.rb │ └── raft.rb └── rust │ ├── Cargo.lock │ ├── Cargo.toml │ ├── README.md │ └── src │ └── bin │ ├── broadcast.rs │ ├── echo.rs │ ├── g_set.rs │ └── lin_kv.rs ├── doc ├── 01-getting-ready │ └── index.md ├── 02-echo │ └── index.md ├── 03-broadcast │ ├── 01-broadcast.md │ ├── 02-performance.md │ ├── broadcast-storm.png │ ├── grid.png │ ├── index.md │ ├── line.png │ └── no-comms.png ├── 04-crdts │ ├── 01-g-set.md │ ├── 02-counters.md │ ├── index.md │ └── latency-partitions.png ├── 05-datomic │ ├── 01-single-node.md │ ├── 02-shared-state.md │ ├── 03-persistent-trees.md │ ├── 04-optimization.md │ ├── g-single-realtime.svg │ ├── g1-realtime.svg │ ├── index.md │ ├── interleaved-keys.png │ ├── linear-slowdown.png │ ├── lww-high-latency.png │ ├── missing-value.png │ ├── not-concurrent.png │ ├── thunk-latency.png │ └── thunk-map.png ├── 06-raft │ ├── 01-key-value.md │ ├── 02-leader-election.md │ ├── 03-replication.md │ ├── 04-committing.md │ ├── final.png │ ├── index.md │ ├── lots-of-failures.png │ ├── no-log-logging.png │ ├── proxy.png │ └── single-node-anomaly.svg ├── promo.png ├── protocol.md ├── results.md ├── services.md └── workloads.md ├── package.sh ├── pkg └── maelstrom ├── project.clj ├── resources ├── errors.edn ├── protocol-intro.md └── workloads-intro.md ├── src └── maelstrom │ ├── checker.clj │ ├── client.clj │ ├── core.clj │ ├── db.clj │ ├── doc.clj │ ├── nemesis.clj │ ├── net.clj │ ├── net │ ├── checker.clj │ ├── journal.clj │ ├── message.clj │ └── viz.clj │ ├── process.clj │ ├── service.clj │ ├── util.clj │ └── workload │ ├── broadcast.clj │ ├── echo.clj │ ├── g_counter.clj │ ├── g_set.clj │ ├── kafka.clj │ ├── lin_kv.clj │ ├── pn_counter.clj │ ├── txn_list_append.clj │ ├── txn_rw_register.clj │ └── unique_ids.clj └── test └── maelstrom ├── core_test.clj ├── service_test.clj └── workload └── pn_counter_test.clj /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /classes 3 | /checkouts 4 | /class 5 | store/ 6 | /pom.xml 7 | /pom.xml.asc 8 | *.log 9 | .*.swp 10 | ~.* 11 | *.jar 12 | *.class 13 | /.lein-* 14 | /.nrepl-port 15 | .hgignore 16 | .hg/ 17 | .DS_Store 18 | .idea 19 | /demo/rust/target 20 | __pycache__/ 21 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | All notable changes to this project will be documented in this file. This change log follows the conventions of [keepachangelog.com](http://keepachangelog.com/). 3 | 4 | ## [Unreleased] 5 | ### Changed 6 | - Add a new arity to `make-widget-async` to provide a different widget shape. 7 | 8 | ## [0.1.1] - 2017-04-10 9 | ### Changed 10 | - Documentation on how to make the widgets. 11 | 12 | ### Removed 13 | - `make-widget-sync` - we're all async, all the time. 14 | 15 | ### Fixed 16 | - Fixed widget maker to keep working when daylight savings switches over. 17 | 18 | ## 0.1.0 - 2017-04-10 19 | ### Added 20 | - Files from the new template. 21 | - Widget maker public API - `make-widget-sync`. 22 | 23 | [Unreleased]: https://github.com/your-name/maelstrom/compare/0.1.1...HEAD 24 | [0.1.1]: https://github.com/your-name/maelstrom/compare/0.1.0...0.1.1 25 | -------------------------------------------------------------------------------- /demo/c++/Makefile: -------------------------------------------------------------------------------- 1 | # Compiler. 2 | CXX = g++ 3 | 4 | # Compiler flags. 5 | CXXFLAGS = -I. -I/opt/homebrew/Cellar/boost/1.82.0_1/include -std=c++17 6 | 7 | # Linker flags. 8 | LDFLAGS = -L/opt/homebrew/Cellar/boost/1.82.0_1/lib 9 | 10 | # Debugging flag. 11 | DEBUG ?= 0 12 | 13 | ifeq ($(DEBUG), 1) 14 | CXXFLAGS += -g 15 | endif 16 | 17 | # Libraries. 18 | LIBS = -lboost_json 19 | 20 | # Source files. 21 | SRC = maelstrom.cpp echo.cpp 22 | 23 | # Object files. 24 | OBJ = $(SRC:.cpp=.o) 25 | 26 | # Target binary. 27 | TARGET = echo 28 | 29 | all: $(TARGET) 30 | 31 | $(TARGET): $(OBJ) 32 | $(CXX) $(LDFLAGS) $(OBJ) -o $(TARGET) $(LIBS) 33 | 34 | %.o: %.cpp 35 | $(CXX) $(CXXFLAGS) -c $< -o $@ 36 | 37 | clean: 38 | rm -f $(OBJ) $(TARGET) 39 | -------------------------------------------------------------------------------- /demo/c++/README.md: -------------------------------------------------------------------------------- 1 | # Maelstrom Node Implementation in C++ 2 | 3 | Welcome to Maelstrom, an advanced node implementation designed to provide a simple and efficient framework for writing C++ code. 4 | 5 | ## Classes 🚀 6 | 7 | **1. `Message`:** This encapsulates the JSON messages that are exchanged between the test binary and the Maelstrom. Serialization and deserialization are taken care of by the Boost.JSON library. 8 | 9 | **2. `Node`:** The medium of message exchange between the test binary and the Maelstrom. It receives messages from the Maelstrom and calls the relevant message handler. Additionally, it offers a means to register a custom message handler. 10 | 11 | **3. `MessageHandler`:** This provides an interface for users to create a custom message handler. 12 | 13 | ## Building the Project 🛠 14 | 15 | Although tested primarily on Mac, the implementation itself is not specific to any particular operating system. 16 | 17 | This project utilizes the [Boost C++ Libraries](https://www.boost.org/) (particularly Boost.JSON), and assumes that Boost is installed on your system. 18 | 19 | You can build the source code using the `Makefile`. Here are a few key variables you might need to adjust: 20 | 21 | - `CXX`: Specifies the C++ compiler. The default is `g++`. 22 | - `CXXFLAGS`: Flags passed to the C++ compiler, including the path to the Boost headers and the `-std=c++17` flag to enable C++17 features. Please adjust the hard-coded Boost headers path (`/opt/homebrew/Cellar/boost/1.82.0_1/include`) to match your system configuration. 23 | - `LDFLAGS`: Flags passed to the linker. The default includes the path to the Boost libraries. Update the hard-coded Boost libraries path (`/opt/homebrew/Cellar/boost/1.82.0_1/lib`) to match your system configuration. 24 | - `DEBUG`: If set to `1`, enables debug flags. 25 | 26 | To build the project: 27 | 28 | ```bash 29 | make 30 | ``` 31 | 32 | To build the project with debug symbols: 33 | 34 | ```bash 35 | make DEBUG=1 36 | ``` 37 | 38 | To clean the build: 39 | 40 | ```bash 41 | make clean 42 | ``` 43 | 44 | ## Running the Echo Server 📡 45 | 46 | The `echo.cpp` file provides an implementation for the `echo` workload. An executable binary is generated upon successful `make` command execution. 47 | 48 | To run this binary against the Maelstrom, issue the following command: 49 | 50 | ```bash 51 | maelstrom test -w echo --bin ./demo/c++/echo --node-count 3 --time-limit 60 52 | ``` 53 | 54 | If you're unfamiliar with the above command, refer to [this section](https://github.com/jepsen-io/maelstrom/blob/main/doc/01-getting-ready/index.md) for further details. 55 | -------------------------------------------------------------------------------- /demo/c++/echo.cpp: -------------------------------------------------------------------------------- 1 | #include "maelstrom.h" 2 | 3 | class EchoMessageHandler : public MessageHandler { 4 | public: 5 | const std::string &name() const final { 6 | static const std::string name = "echo"; 7 | return name; 8 | } 9 | 10 | void handle(const Message &request) final { 11 | constexpr const char *msgType = "echo_ok"; 12 | 13 | auto sender = request.getRecipient(); 14 | auto recipient = request.getSender(); 15 | auto msgBody = request.getBody(); 16 | auto msgId = boost::none; 17 | auto msgInReplyTo = request.getMsgId(); 18 | 19 | Message response{sender, recipient, msgType, msgBody, msgId, msgInReplyTo}; 20 | response.send(); 21 | } 22 | }; 23 | 24 | int main() { 25 | Node node; 26 | node.registerMessageHandler(std::make_unique()); 27 | node.run(); 28 | return 0; 29 | } -------------------------------------------------------------------------------- /demo/c++/maelstrom.cpp: -------------------------------------------------------------------------------- 1 | #include "maelstrom.h" 2 | #include 3 | #include 4 | 5 | // 6 | // Message 7 | // 8 | Message::Message(const std::string &strMessage) { 9 | auto jsonValue = boost::json::parse(strMessage); 10 | if (!jsonValue.is_object()) { 11 | raiseRuntimeError("Invalid message: ", strMessage); 12 | } 13 | 14 | auto jsonObject = jsonValue.as_object(); 15 | 16 | _sender = jsonObject.at("src").as_string().c_str(); 17 | _recipient = jsonObject.at("dest").as_string().c_str(); 18 | _body = jsonObject.at("body").as_object(); 19 | _type = _body.at("type").as_string().c_str(); 20 | 21 | if (_body.contains("msg_id")) { 22 | _msgId = _body.at("msg_id").as_int64(); 23 | } 24 | 25 | if (_body.contains("in_reply_to")) { 26 | _inReplyTo = _body.at("in_reply_to").as_int64(); 27 | } 28 | } 29 | 30 | Message::Message(const std::string &sender, const std::string &recipent, 31 | const std::string &type, const boost::json::object &body, 32 | const boost::optional &msgId, 33 | const boost::optional &inReplyTo) 34 | : _sender(sender), _recipient(recipent), _type(type), _body(body), 35 | _msgId(msgId), _inReplyTo(inReplyTo) { 36 | _body["type"] = _type; 37 | 38 | if (msgId && inReplyTo) { 39 | raiseRuntimeError("'msg_Id' and 'in_reply_to' cannot be set together"); 40 | } 41 | 42 | if (_msgId) { 43 | _body["msg_id"] = *_msgId; 44 | if (_body.contains("in_reply_to")) { 45 | _body.erase("in_reply_to"); 46 | } 47 | } 48 | 49 | if (_inReplyTo) { 50 | _body["in_reply_to"] = *_inReplyTo; 51 | if (_body.contains("msg_id")) { 52 | _body.erase("msg_id"); 53 | } 54 | } 55 | } 56 | 57 | std::string Message::getSender() const { return _sender; } 58 | 59 | std::string Message::getRecipient() const { return _recipient; } 60 | 61 | std::string Message::getType() const { return _type; } 62 | 63 | const boost::json::object &Message::getBody() const { return _body; } 64 | 65 | boost::optional Message::getMsgId() const { return _msgId; } 66 | 67 | boost::optional Message::getInReplyTo() const { return _inReplyTo; } 68 | 69 | void Message::send() const { 70 | boost::json::object jsonMessage; 71 | jsonMessage["src"] = _sender; 72 | jsonMessage["dest"] = _recipient; 73 | jsonMessage["body"] = _body; 74 | std::cout << jsonMessage << std::endl; 75 | } 76 | 77 | // 78 | // Node 79 | // 80 | void Node::run() { 81 | while (true) { 82 | std::string inputLine; 83 | std::getline(std::cin, inputLine); 84 | if (inputLine.empty()) { 85 | break; 86 | } 87 | 88 | Message request{inputLine}; 89 | handleMessage(request); 90 | } 91 | } 92 | 93 | void Node::registerMessageHandler(MessageHandlerPtr msgHandler) { 94 | _messageHandlers[msgHandler->name()] = std::move(msgHandler); 95 | } 96 | 97 | void Node::handleMessage(const Message &request) { 98 | auto msgType = request.getType(); 99 | 100 | if (msgType == "init") { 101 | _init(request); 102 | return; 103 | } 104 | 105 | if (auto handlersIter = _messageHandlers.find(msgType); 106 | handlersIter != _messageHandlers.end()) { 107 | std::async(std::launch::async, 108 | [&]() { handlersIter->second->handle(request); }); 109 | } else { 110 | raiseRuntimeError("No handler found for message type", msgType); 111 | } 112 | } 113 | 114 | void Node::_init(const Message &request) { 115 | if (_nodeId) { 116 | raiseRuntimeError("Node: ", *_nodeId, " already initialized"); 117 | } 118 | 119 | auto requestBody = request.getBody(); 120 | _nodeId = requestBody.at("node_id").as_string().c_str(); 121 | 122 | _peerIds = [&]() { 123 | std::vector peers; 124 | for (const auto &item : requestBody.at("node_ids").as_array()) { 125 | peers.push_back(item.as_string().c_str()); 126 | } 127 | return peers; 128 | }(); 129 | 130 | constexpr const char *msgType = "init_ok"; 131 | auto sender = request.getRecipient(); 132 | auto recipient = request.getSender(); 133 | auto responseBody = boost::json::object(); 134 | auto msgId = boost::none; 135 | auto msgInReplyTo = request.getMsgId(); 136 | 137 | Message response{sender, recipient, msgType, 138 | responseBody, msgId, msgInReplyTo}; 139 | response.send(); 140 | } 141 | -------------------------------------------------------------------------------- /demo/c++/maelstrom.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | using namespace std; 13 | 14 | // Helper function for raising runtime errors. 15 | template void raiseRuntimeError(Args &&...args) { 16 | std::stringstream errorStream; 17 | (errorStream << ... << args); 18 | auto error = errorStream.str(); 19 | throw std::runtime_error(error); 20 | } 21 | 22 | /** 23 | * Message class that is used to encapsulate 'Maelstrom' messages. 24 | */ 25 | class Message { 26 | public: 27 | Message(const std::string &strMessage); 28 | 29 | Message(const std::string &sender, const std::string &recipent, 30 | const std::string &type, 31 | const boost::json::object &body = boost::json::object(), 32 | const boost::optional &msgId = boost::none, 33 | const boost::optional &inReplyTo = boost::none); 34 | 35 | // Returns the sender of the message. 36 | std::string getSender() const; 37 | 38 | // Returns the recipient of the message. 39 | std::string getRecipient() const; 40 | 41 | // Returns the type of the message. 42 | std::string getType() const; 43 | 44 | // Returns the body of the message. 45 | const boost::json::object &getBody() const; 46 | 47 | // Returns the message-id if present, otherwise returns 'boost::none'. 48 | boost::optional getMsgId() const; 49 | 50 | // Returns the reply message-id if present, otherwise returns 'boost::none'. 51 | boost::optional getInReplyTo() const; 52 | 53 | // Sends the message to the 'Maelstrom'. 54 | void send() const; 55 | 56 | private: 57 | // Field 'src' of the message. 58 | std::string _sender; 59 | 60 | // Field 'dest' of the message. 61 | std::string _recipient; 62 | 63 | // Field 'type' of the message. 64 | std::string _type; 65 | 66 | // Field 'body' of the message. 67 | boost::json::object _body; 68 | 69 | // Field 'msg_id' of the message if present. 70 | boost::optional _msgId; 71 | 72 | // Field 'in_reply_to' of the message if present. 73 | boost::optional _inReplyTo; 74 | }; 75 | 76 | /** 77 | * Abstract base class for writing custom message handler. 78 | */ 79 | class MessageHandler { 80 | public: 81 | virtual ~MessageHandler() = default; 82 | 83 | // Returns the type of message associated with this handler. 84 | virtual const std::string &name() const = 0; 85 | 86 | // Handles the provided message which is guaranteed to be of the expected 87 | // type. 88 | virtual void handle(const Message &message) = 0; 89 | }; 90 | 91 | using MessageHandlerPtr = std::unique_ptr; 92 | 93 | /** 94 | * Node class that interacts with the 'Maelstrom'. 95 | */ 96 | class Node { 97 | public: 98 | // Starts receiving messages from the 'Maelstrom' and dispatches them to the 99 | // appropriate message handler. The user should call this method. 100 | void run(); 101 | 102 | // Registers a message handler with the node. The user should call this method 103 | // before calling 'run()'. 104 | void registerMessageHandler(MessageHandlerPtr msgHandler); 105 | 106 | private: 107 | // Initializes the node with 'nodeId' and 'peerIds'. 108 | void _init(const Message &message); 109 | 110 | // Calls the appropriate message handler for the provided message. 111 | void handleMessage(const Message &message); 112 | 113 | // Maps message type to message handler. 114 | std::unordered_map _messageHandlers; 115 | 116 | // Node ID of this node. 117 | boost::optional _nodeId; 118 | 119 | // Node IDs of the peers of this node. 120 | std::vector _peerIds; 121 | }; 122 | -------------------------------------------------------------------------------- /demo/clojure/echo.clj: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bb 2 | 3 | (ns maelstrom.echo 4 | (:gen-class) 5 | (:require 6 | [cheshire.core :as json])) 7 | 8 | 9 | ;;;;;;;;;;;;;;;;;;; Util functions ;;;;;;;;;;;;;;;;;;; 10 | 11 | ;;;;;; Input pre-processing functions ;;;;;; 12 | 13 | (defn- process-stdin 14 | "Read lines from the stdin and calls the handler" 15 | [handler] 16 | (doseq [line (line-seq (java.io.BufferedReader. *in*))] 17 | (handler line))) 18 | 19 | 20 | (defn- parse-json 21 | "Parse the received input as json" 22 | [input] 23 | (try 24 | (json/parse-string input true) 25 | (catch Exception e 26 | nil))) 27 | 28 | 29 | ;;;;;; Output Generating functions ;;;;;; 30 | 31 | (defn- generate-json 32 | "Generate json string from input" 33 | [input] 34 | (json/generate-string input)) 35 | 36 | 37 | (defn- printerr 38 | "Print the received input to stderr" 39 | [input] 40 | (binding [*out* *err*] 41 | (println input))) 42 | 43 | 44 | (defn- printout 45 | "Print the received input to stdout" 46 | [input] 47 | (println input)) 48 | 49 | 50 | (defn reply 51 | ([src dest body] 52 | {:src src 53 | :dest dest 54 | :body body})) 55 | 56 | 57 | (def node-id (atom "")) 58 | (def next-message-id (atom 0)) 59 | 60 | 61 | (defn- process-request 62 | [input] 63 | (let [body (:body input) 64 | r-body {:msg_id (swap! next-message-id inc) 65 | :in_reply_to (:msg_id body)}] 66 | (case (:type body) 67 | "init" 68 | (do 69 | (reset! node-id (:node_id body)) 70 | (reply @node-id 71 | (:src input) 72 | (assoc r-body :type "init_ok"))) 73 | "echo" 74 | (reply @node-id 75 | (:src input) 76 | (assoc r-body 77 | :type "echo_ok" 78 | :echo (:echo body)))))) 79 | 80 | 81 | (defn -main 82 | "Read transactions from stdin and send output to stdout" 83 | [] 84 | (process-stdin (comp printout 85 | generate-json 86 | process-request 87 | parse-json))) 88 | 89 | 90 | (-main) 91 | -------------------------------------------------------------------------------- /demo/clojure/flake_ids.clj: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bb 2 | 3 | (load-file (clojure.string/replace *file* #"/[^/]+$" "/node.clj")) 4 | 5 | (ns maelstrom.demo.flake-ids 6 | "A totally-available Flake ID generator for the unique-ids workload. Uses the 7 | local node clock, a counter, and the node ID." 8 | (:require [maelstrom.demo.node :as node])) 9 | 10 | (def flake-state 11 | "An atom with both a last-generated time and a counter for IDs generated at 12 | that timestamp." 13 | (atom {:time 0 14 | :count 0})) 15 | 16 | (node/defhandler generate 17 | [{:keys [body] :as req}] 18 | (let [; Intentionally use low precision here so that we stress the counter 19 | ; system 20 | time (long (/ (System/currentTimeMillis) 1000)) 21 | {:keys [time count]} (swap! flake-state 22 | (fn [fs] 23 | (let [time (max time (:time fs)) 24 | count (if (= time (:time fs)) 25 | (inc (:count fs)) 26 | 0)] 27 | {:time time 28 | :count count})))] 29 | (node/reply! req 30 | {:type :generate_ok 31 | :id [time count @node/node-id]}))) 32 | 33 | (node/start!) 34 | -------------------------------------------------------------------------------- /demo/clojure/gcounter.clj: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bb 2 | 3 | (ns maelstrom.gcounter 4 | (:gen-class) 5 | (:require 6 | [cheshire.core :as json] 7 | [clojure.walk :as walk])) 8 | 9 | 10 | ;;;;;;;;;;;;;;;;;;; Util functions ;;;;;;;;;;;;;;;;;;; 11 | 12 | ;;;;;; Input pre-processing functions ;;;;;; 13 | 14 | 15 | (defn- process-stdin 16 | "Read lines from the stdin and calls the handler" 17 | [handler] 18 | (doseq [line (line-seq (java.io.BufferedReader. *in*))] 19 | (handler line))) 20 | 21 | 22 | (defn- parse-json 23 | "Parse the received input as json" 24 | [input] 25 | (try 26 | (json/parse-string input true) 27 | (catch Exception e 28 | nil))) 29 | 30 | 31 | ;;;;;; Output Generating functions ;;;;;; 32 | 33 | (defn- generate-json 34 | "Generate json string from input" 35 | [input] 36 | (when input 37 | (json/generate-string input))) 38 | 39 | 40 | (let [l (Object.)] 41 | (defn- printerr 42 | "Print the received input to stderr" 43 | [input] 44 | (locking l 45 | (binding [*out* *err*] 46 | (println input))))) 47 | 48 | 49 | (let [l (Object.)] 50 | (defn- printout 51 | "Print the received input to stdout" 52 | [input] 53 | (when input 54 | (locking l 55 | (println input))))) 56 | 57 | 58 | (defprotocol CRDT 59 | "A protocol which defines behavior of a 60 | CRDT type" 61 | 62 | (combine [this other]) 63 | 64 | (add [this element]) 65 | 66 | (serialize [this]) 67 | 68 | (to-val [this])) 69 | 70 | 71 | (defrecord GCounter 72 | [data] 73 | 74 | CRDT 75 | 76 | (combine 77 | [this other] 78 | (assoc this :data (merge-with max data other))) 79 | 80 | 81 | (add 82 | [this element] 83 | (update-in this 84 | [:data (:node_id element)] 85 | (fnil + 0) 86 | (:value element))) 87 | 88 | 89 | (to-val 90 | [this] 91 | (reduce + 0 (vals data))) 92 | 93 | 94 | (serialize 95 | [this] 96 | data)) 97 | 98 | 99 | (defrecord PNCounter 100 | [inc dec] 101 | 102 | CRDT 103 | 104 | (combine 105 | [this other] 106 | (assoc this 107 | :inc (combine inc (get other "inc")) 108 | :dec (combine dec (get other "dec")))) 109 | 110 | 111 | (add 112 | [this element] 113 | (if (< 0 (:value element)) 114 | (assoc this :inc (add inc element)) 115 | (assoc this :dec (add dec (update element 116 | :value 117 | #(Math/abs %)))))) 118 | 119 | 120 | (to-val 121 | [this] 122 | (- (to-val inc) (to-val dec))) 123 | 124 | 125 | (serialize 126 | [this] 127 | {:inc (serialize inc) 128 | :dec (serialize dec)})) 129 | 130 | 131 | (def node-id (atom "")) 132 | (def node-nbrs (atom [])) 133 | (def gset (atom nil)) 134 | (def next-message-id (atom 0)) 135 | 136 | 137 | (defn- reply 138 | ([src dest body] 139 | {:src src 140 | :dest dest 141 | :body body})) 142 | 143 | 144 | (defn- send! 145 | ([input] 146 | (-> input 147 | generate-json 148 | printout)) 149 | ([src dest body] 150 | (send! (reply src dest body)))) 151 | 152 | 153 | (defn- replicate-loop 154 | [] 155 | (future 156 | (try 157 | (loop [] 158 | (doseq [n @node-nbrs] 159 | (send! (reply @node-id n {:value (serialize @gset) 160 | :type "replicate"}))) 161 | (Thread/sleep 5000) 162 | (recur)) 163 | (catch Exception e 164 | (printerr e))))) 165 | 166 | 167 | (defn- process-request 168 | [input] 169 | (let [body (:body input) 170 | r-body {:msg_id (swap! next-message-id inc) 171 | :in_reply_to (:msg_id body)} 172 | nid (:node_id body) 173 | nids (:node_ids body)] 174 | (case (:type body) 175 | "init" 176 | (do 177 | (reset! node-id nid) 178 | (reset! node-nbrs nids) 179 | (reply @node-id 180 | (:src input) 181 | (assoc r-body :type "init_ok"))) 182 | 183 | "add" 184 | (do 185 | (swap! gset add {:node_id @node-id 186 | :value (:delta body)}) 187 | (when (:msg_id body) 188 | (reply @node-id 189 | (:src input) 190 | (assoc r-body 191 | :type "add_ok")))) 192 | 193 | "replicate" 194 | (do 195 | ;; stringify keys 196 | (swap! gset combine (walk/stringify-keys (:value body))) 197 | nil) 198 | 199 | "read" 200 | (reply @node-id 201 | (:src input) 202 | (assoc r-body 203 | :type "read_ok" 204 | :value (to-val @gset)))))) 205 | 206 | 207 | (defn- gcounter [] 208 | (->GCounter {})) 209 | 210 | 211 | (defn- pncounter [] 212 | (->PNCounter (gcounter) (gcounter))) 213 | 214 | 215 | (defn -main 216 | "Read transactions from stdin and send output to stdout" 217 | [] 218 | (replicate-loop) 219 | ;; if you want to run the g-counter workload, 220 | ;; create the gcounter CRDT 221 | ;;(reset! gset (gcounter)) 222 | ;; if you want to run the pn-counter workload, 223 | ;; create the pncounter CRDT 224 | (reset! gset (pncounter)) 225 | (process-stdin (comp printout 226 | generate-json 227 | process-request 228 | parse-json))) 229 | 230 | 231 | (-main) 232 | -------------------------------------------------------------------------------- /demo/clojure/gossip.clj: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bb 2 | 3 | (ns maelstrom.gossip 4 | (:gen-class) 5 | (:require 6 | [cheshire.core :as json])) 7 | 8 | 9 | ;;;;;;;;;;;;;;;;;;; Util functions ;;;;;;;;;;;;;;;;;;; 10 | 11 | ;;;;;; Input pre-processing functions ;;;;;; 12 | 13 | 14 | (defn- process-stdin 15 | "Read lines from the stdin and calls the handler" 16 | [handler] 17 | (doseq [line (line-seq (java.io.BufferedReader. *in*))] 18 | (handler line))) 19 | 20 | 21 | (defn- parse-json 22 | "Parse the received input as json" 23 | [input] 24 | (try 25 | (json/parse-string input true) 26 | (catch Exception e 27 | nil))) 28 | 29 | 30 | ;;;;;; Output Generating functions ;;;;;; 31 | 32 | (defn- generate-json 33 | "Generate json string from input" 34 | [input] 35 | (when input 36 | (json/generate-string input))) 37 | 38 | 39 | (let [l (Object.)] 40 | (defn- printerr 41 | "Print the received input to stderr" 42 | [input] 43 | (locking l 44 | (binding [*out* *err*] 45 | (println input))))) 46 | 47 | 48 | (let [l (Object.)] 49 | (defn- printout 50 | "Print the received input to stdout" 51 | [input] 52 | (when input 53 | (locking l 54 | (println input))))) 55 | 56 | 57 | (def node-id (atom "")) 58 | (def node-nbrs (atom [])) 59 | (def messages (atom #{})) 60 | (def gossips (atom {})) 61 | (def next-message-id (atom 0)) 62 | 63 | 64 | (defn- reply 65 | ([src dest body] 66 | {:src src 67 | :dest dest 68 | :body body})) 69 | 70 | 71 | (defn- send! 72 | ([input] 73 | (-> input 74 | generate-json 75 | printout)) 76 | ([src dest body] 77 | (send! (reply src dest body)))) 78 | 79 | 80 | (defn- gossip-loop 81 | [] 82 | (future 83 | (try 84 | (loop [] 85 | (doseq [g @gossips] 86 | (send! (second g))) 87 | (Thread/sleep 1000) 88 | (recur)) 89 | (catch Exception e 90 | (printerr e))))) 91 | 92 | 93 | (defn- process-request 94 | [input] 95 | (let [body (:body input) 96 | r-body {:msg_id (swap! next-message-id inc) 97 | :in_reply_to (:msg_id body)} 98 | nid (:node_id body)] 99 | (case (:type body) 100 | "init" 101 | (do 102 | (reset! node-id nid) 103 | (reply @node-id 104 | (:src input) 105 | (assoc r-body :type "init_ok"))) 106 | 107 | "topology" 108 | (do 109 | (reset! node-nbrs (get-in input [:body 110 | :topology 111 | (keyword @node-id)])) 112 | (reply @node-id 113 | (:src input) 114 | (assoc r-body 115 | :type "topology_ok"))) 116 | 117 | "broadcast" 118 | (do 119 | (when-not (@messages (:message body)) 120 | (doseq [n @node-nbrs 121 | ;; also don't broadcast to sender 122 | :when (not= n (:src input))] 123 | ;; here we need to distinguish between msg-id received 124 | ;; directly from a broadcast message which will be :msg_id 125 | ;; and msg-id received from a peer as part of gossip. 126 | ;; We can receive gossip with msg-id 1 and message 1 127 | ;; and later receive broadcast with msg-id 1 and message 3 128 | ;; Also, the msg-id we send here has to be used to 129 | ;; record the message for retries which is then removed 130 | ;; when we receive gossip_ok message 131 | (let [msg-id (or (:msg_id body) 132 | (str "g:" (:g_id body))) 133 | msg {:src @node-id 134 | :dest n 135 | :body {:type "broadcast" 136 | :message (:message body) 137 | :g_id msg-id}}] 138 | (swap! gossips assoc (str n "::" msg-id) msg) 139 | ;; we have to explicitly send this out 140 | ;; because just calling reply only generates a 141 | ;; map which the outer loop prints out 142 | (send! msg)))) 143 | (swap! messages conj (:message body)) 144 | (if (:msg_id body) 145 | (reply @node-id 146 | (:src input) 147 | (assoc r-body 148 | :type "broadcast_ok")) 149 | (when (:g_id body) 150 | (reply @node-id 151 | (:src input) 152 | {:type "gossip_ok" 153 | :g_id (:g_id body)})))) 154 | 155 | "gossip_ok" 156 | (do 157 | (swap! gossips dissoc (str (:src input) "::" (:g_id body))) 158 | nil) 159 | 160 | "read" 161 | (reply @node-id 162 | (:src input) 163 | (assoc r-body 164 | :type "read_ok" 165 | :messages @messages))))) 166 | 167 | 168 | (defn -main 169 | "Read transactions from stdin and send output to stdout" 170 | [] 171 | (gossip-loop) 172 | (process-stdin (comp printout 173 | generate-json 174 | process-request 175 | parse-json))) 176 | 177 | 178 | (-main) 179 | -------------------------------------------------------------------------------- /demo/clojure/gset.clj: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bb 2 | 3 | (ns maelstrom.gset 4 | (:gen-class) 5 | (:require 6 | [cheshire.core :as json] 7 | [clojure.set :as set])) 8 | 9 | 10 | ;;;;;;;;;;;;;;;;;;; Util functions ;;;;;;;;;;;;;;;;;;; 11 | 12 | ;;;;;; Input pre-processing functions ;;;;;; 13 | 14 | 15 | (defn- process-stdin 16 | "Read lines from the stdin and calls the handler" 17 | [handler] 18 | (doseq [line (line-seq (java.io.BufferedReader. *in*))] 19 | (handler line))) 20 | 21 | 22 | (defn- parse-json 23 | "Parse the received input as json" 24 | [input] 25 | (try 26 | (json/parse-string input true) 27 | (catch Exception e 28 | nil))) 29 | 30 | 31 | ;;;;;; Output Generating functions ;;;;;; 32 | 33 | (defn- generate-json 34 | "Generate json string from input" 35 | [input] 36 | (when input 37 | (json/generate-string input))) 38 | 39 | 40 | (let [l (Object.)] 41 | (defn- printerr 42 | "Print the received input to stderr" 43 | [input] 44 | (locking l 45 | (binding [*out* *err*] 46 | (println input))))) 47 | 48 | 49 | (let [l (Object.)] 50 | (defn- printout 51 | "Print the received input to stdout" 52 | [input] 53 | (when input 54 | (locking l 55 | (println input))))) 56 | 57 | 58 | (defprotocol CRDT 59 | "A protocol which defines behavior of a 60 | CRDT type" 61 | 62 | (combine [this other]) 63 | 64 | (add [this element]) 65 | 66 | (serialize [this]) 67 | 68 | (to-val [this])) 69 | 70 | 71 | (defrecord GSet 72 | [data] 73 | 74 | CRDT 75 | 76 | (combine 77 | [this other] 78 | (assoc this :data (set/union data other))) 79 | 80 | 81 | (add 82 | [this element] 83 | (assoc this :data (conj data element))) 84 | 85 | 86 | (serialize 87 | [this] 88 | (vec data)) 89 | 90 | (to-val 91 | [this] 92 | data)) 93 | 94 | 95 | (def node-id (atom "")) 96 | (def node-nbrs (atom [])) 97 | (def gset (atom nil)) 98 | (def next-message-id (atom 0)) 99 | 100 | 101 | (defn- reply 102 | ([src dest body] 103 | {:src src 104 | :dest dest 105 | :body body})) 106 | 107 | 108 | (defn- send! 109 | ([input] 110 | (-> input 111 | generate-json 112 | printout)) 113 | ([src dest body] 114 | (send! (reply src dest body)))) 115 | 116 | 117 | (defn- replicate-loop 118 | [] 119 | (future 120 | (try 121 | (loop [] 122 | (doseq [n @node-nbrs] 123 | (send! (reply @node-id n {:value (serialize @gset) 124 | :type "replicate"}))) 125 | (Thread/sleep 5000) 126 | (recur)) 127 | (catch Exception e 128 | (printerr e))))) 129 | 130 | 131 | (defn- process-request 132 | [input] 133 | (let [body (:body input) 134 | r-body {:msg_id (swap! next-message-id inc) 135 | :in_reply_to (:msg_id body)} 136 | nid (:node_id body) 137 | nids (:node_ids body)] 138 | (case (:type body) 139 | "init" 140 | (do 141 | (reset! node-id nid) 142 | (reset! node-nbrs nids) 143 | (reply @node-id 144 | (:src input) 145 | (assoc r-body :type "init_ok"))) 146 | 147 | "add" 148 | (do 149 | (swap! gset add (:element body)) 150 | (when (:msg_id body) 151 | (reply @node-id 152 | (:src input) 153 | (assoc r-body 154 | :type "add_ok")))) 155 | 156 | "replicate" 157 | (do 158 | (swap! gset combine (set (:value body))) 159 | nil) 160 | 161 | "read" 162 | (reply @node-id 163 | (:src input) 164 | (assoc r-body 165 | :type "read_ok" 166 | :value (to-val @gset)))))) 167 | 168 | 169 | (defn -main 170 | "Read transactions from stdin and send output to stdout" 171 | [] 172 | (replicate-loop) 173 | (reset! gset (->GSet #{})) 174 | (process-stdin (comp printout 175 | generate-json 176 | process-request 177 | parse-json))) 178 | 179 | 180 | (-main) 181 | -------------------------------------------------------------------------------- /demo/clojure/txn_rw_register_no_isolation.clj: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bb 2 | 3 | (load-file (clojure.string/replace *file* #"/[^/]+$" "/node.clj")) 4 | 5 | (ns maelstrom.demo.txn-rw-register-no-isolation 6 | "A single-node, completely un-isolated rw-register transaction system. Useful 7 | for demonstrating safety violations." 8 | (:require [maelstrom.demo.node :as node])) 9 | 10 | (def state 11 | "An atom of a map of keys to values." 12 | (atom {})) 13 | 14 | (defn apply-txn! 15 | "Takes a state and a txn. Applies the txn to the state without any isolation, 16 | mutating the state and returning txn'. Deliberately introduces sleep 17 | statements to increase the chances of interleaving with other txns." 18 | [txn] 19 | ; Zip through the txn, applying each mop 20 | (reduce (fn [txn' [f k v :as mop]] 21 | (Thread/sleep 1) 22 | (case f 23 | "r" (conj txn' [f k (get @state k)]) 24 | "w" (do (swap! state assoc k v) 25 | (conj txn' mop)))) 26 | [] 27 | txn)) 28 | 29 | (node/defhandler txn 30 | "When a transaction arrives, apply it to the local state." 31 | [{:keys [body] :as req}] 32 | (node/reply! req {:type "txn_ok" 33 | :txn (apply-txn! (:txn body))})) 34 | 35 | (node/start!) 36 | -------------------------------------------------------------------------------- /demo/go/README.md: -------------------------------------------------------------------------------- 1 | maelstrom-go 2 | ============ 3 | 4 | This is a Go implementation of the Maelstrom Node. This provides basic message 5 | handling, an event loop, & a client interface to the key/value store. It's a 6 | good starting point for implementing a Maelstrom node as it helps to avoid a 7 | lot of boilerplate. 8 | 9 | ## Usage 10 | 11 | Binaries run by `maelstrom` need to be referenced by absolute or relative path. 12 | The easiest way to use Go with Maelstrom is to `go install` and then specify 13 | the relative path to the `--bin` flag: 14 | 15 | ```sh 16 | $ cd /path/to/maelstrom-echo 17 | $ go install . 18 | $ maelstrom test --bin ~/go/bin/maelstrom-echo ... 19 | ``` 20 | 21 | -------------------------------------------------------------------------------- /demo/go/cmd/maelstrom-echo/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "os" 7 | 8 | maelstrom "github.com/jepsen-io/maelstrom/demo/go" 9 | ) 10 | 11 | func main() { 12 | n := maelstrom.NewNode() 13 | 14 | // Register a handler for the "echo" message that responds with an "echo_ok". 15 | n.Handle("echo", func(msg maelstrom.Message) error { 16 | // Unmarshal the message body as an loosely-typed map. 17 | var body map[string]any 18 | if err := json.Unmarshal(msg.Body, &body); err != nil { 19 | return err 20 | } 21 | 22 | // Update the message type. 23 | body["type"] = "echo_ok" 24 | 25 | // Echo the original message back with the updated message type. 26 | return n.Reply(msg, body) 27 | }) 28 | 29 | // Execute the node's message loop. This will run until STDIN is closed. 30 | if err := n.Run(); err != nil { 31 | log.Printf("ERROR: %s", err) 32 | os.Exit(1) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /demo/go/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/jepsen-io/maelstrom/demo/go 2 | 3 | go 1.19 4 | -------------------------------------------------------------------------------- /demo/go/kv.go: -------------------------------------------------------------------------------- 1 | package maelstrom 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | ) 7 | 8 | // Types of key/value stores. 9 | const ( 10 | LinKV = "lin-kv" 11 | SeqKV = "seq-kv" 12 | LWWKV = "lww-kv" 13 | ) 14 | 15 | // KV represents a client to the key/value store service. 16 | type KV struct { 17 | typ string 18 | node *Node 19 | } 20 | 21 | // NewKV returns a new instance a KV client for a node. 22 | func NewKV(typ string, node *Node) *KV { 23 | return &KV{ 24 | typ: typ, 25 | node: node, 26 | } 27 | } 28 | 29 | // NewLinKV returns a client to the linearizable key/value store. 30 | func NewLinKV(node *Node) *KV { return NewKV(LinKV, node) } 31 | 32 | // NewSeqKV returns a client to the sequential key/value store. 33 | func NewSeqKV(node *Node) *KV { return NewKV(SeqKV, node) } 34 | 35 | // NewLWWKV returns a client to the last-write-wins key/value store. 36 | func NewLWWKV(node *Node) *KV { return NewKV(LWWKV, node) } 37 | 38 | // Read returns the value for a given key in the key/value store. 39 | // Returns an *RPCError error with a KeyDoesNotExist code if the key does not exist. 40 | func (kv *KV) Read(ctx context.Context, key string) (any, error) { 41 | resp, err := kv.read(ctx, key) 42 | if err != nil { 43 | return nil, err 44 | } 45 | 46 | // Parse read_ok specific data in response message. 47 | var body kvReadOKMessageBody 48 | if err := json.Unmarshal(resp.Body, &body); err != nil { 49 | return nil, err 50 | } 51 | 52 | // Convert numbers to integers since that's what maelstrom workloads use. 53 | switch v := body.Value.(type) { 54 | case float64: 55 | return int(v), nil 56 | default: 57 | return v, nil 58 | } 59 | } 60 | 61 | // ReadInto reads the value of a key in the key/value store and store it in the value pointed by v. 62 | // Returns an *RPCError error with a KeyDoesNotExist code if the key does not exist. 63 | func (kv *KV) ReadInto(ctx context.Context, key string, v any) error { 64 | resp, err := kv.read(ctx, key) 65 | if err != nil { 66 | return err 67 | } 68 | 69 | var body kvReadOKMessageBody 70 | body.Value = v 71 | return json.Unmarshal(resp.Body, &body) 72 | } 73 | 74 | // ReadInt reads the value of a key in the key/value store as an int. 75 | func (kv *KV) ReadInt(ctx context.Context, key string) (int, error) { 76 | v, err := kv.Read(ctx, key) 77 | i, _ := v.(int) 78 | return i, err 79 | } 80 | 81 | // Write overwrites the value for a given key in the key/value store. 82 | func (kv *KV) Write(ctx context.Context, key string, value any) error { 83 | _, err := kv.node.SyncRPC(ctx, kv.typ, kvWriteMessageBody{ 84 | MessageBody: MessageBody{Type: "write"}, 85 | Key: key, 86 | Value: value, 87 | }) 88 | return err 89 | } 90 | 91 | // CompareAndSwap updates the value for a key if its current value matches the 92 | // previous value. Creates the key if createIfNotExists is true. 93 | // 94 | // Returns an *RPCError with a code of PreconditionFailed if the previous value 95 | // does not match. Return a code of KeyDoesNotExist if the key did not exist. 96 | func (kv *KV) CompareAndSwap(ctx context.Context, key string, from, to any, createIfNotExists bool) error { 97 | _, err := kv.node.SyncRPC(ctx, kv.typ, kvCASMessageBody{ 98 | MessageBody: MessageBody{Type: "cas"}, 99 | Key: key, 100 | From: from, 101 | To: to, 102 | CreateIfNotExists: createIfNotExists, 103 | }) 104 | return err 105 | } 106 | 107 | func (kv *KV) read(ctx context.Context, key string) (Message, error) { 108 | resp, err := kv.node.SyncRPC(ctx, kv.typ, kvReadMessageBody{ 109 | MessageBody: MessageBody{Type: "read"}, 110 | Key: key, 111 | }) 112 | if err != nil { 113 | return Message{}, err 114 | } 115 | return resp, nil 116 | } 117 | 118 | // kvReadMessageBody represents the body for the KV "read" message. 119 | type kvReadMessageBody struct { 120 | MessageBody 121 | Key string `json:"key"` 122 | } 123 | 124 | // kvReadOKMessageBody represents the response body for the KV "read_ok" message. 125 | type kvReadOKMessageBody struct { 126 | MessageBody 127 | Value any `json:"value"` 128 | } 129 | 130 | // kvWriteMessageBody represents the body for the KV "write" message. 131 | type kvWriteMessageBody struct { 132 | MessageBody 133 | Key string `json:"key"` 134 | Value any `json:"value"` 135 | } 136 | 137 | // kvCASMessageBody represents the body for the KV "cas" message. 138 | type kvCASMessageBody struct { 139 | MessageBody 140 | Key string `json:"key"` 141 | From any `json:"from"` 142 | To any `json:"to"` 143 | CreateIfNotExists bool `json:"create_if_not_exists,omitempty"` 144 | } 145 | -------------------------------------------------------------------------------- /demo/go/kv_test.go: -------------------------------------------------------------------------------- 1 | package maelstrom_test 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "testing" 7 | "time" 8 | 9 | maelstrom "github.com/jepsen-io/maelstrom/demo/go" 10 | ) 11 | 12 | func TestKVReadStruct(t *testing.T) { 13 | type testPayload struct { 14 | Counter int 15 | } 16 | t.Run("OK", func(t *testing.T) { 17 | n, stdin, stdout := newNode(t) 18 | kv := maelstrom.NewSeqKV(n) 19 | initNode(t, n, "n1", []string{"n1"}, stdin, stdout) 20 | 21 | respCh := make(chan testPayload) 22 | errorCh := make(chan error) 23 | go func() { 24 | var p testPayload 25 | err := kv.ReadInto(context.Background(), "foo", &p) 26 | if err != nil { 27 | errorCh <- err 28 | return 29 | } 30 | respCh <- p 31 | }() 32 | 33 | // Ensure RPC request is received by the network. 34 | if line, err := stdout.ReadString('\n'); err != nil { 35 | t.Fatal(err) 36 | } else if got, want := line, `{"src":"n1","dest":"seq-kv","body":{"key":"foo","msg_id":1,"type":"read"}}`+"\n"; got != want { 37 | t.Fatalf("response=%s, want %s", got, want) 38 | } 39 | 40 | // Write response message back to node. 41 | if _, err := stdin.Write([]byte(`{"src":"seq-kv","dest":"n1","body":{"type":"read_ok","value":{"Counter":13},"msg_id":2,"in_reply_to":1}}` + "\n")); err != nil { 42 | t.Fatal(err) 43 | } 44 | 45 | select { 46 | case p := <-respCh: 47 | if got, want := p.Counter, 13; got != want { 48 | t.Fatalf("counter=%d, want %d", got, want) 49 | } 50 | case err := <-errorCh: 51 | t.Fatal(err) 52 | case <-time.After(5 * time.Second): 53 | t.Fatal("timeout waiting for RPC response") 54 | } 55 | }) 56 | t.Run("RPCError", func(t *testing.T) { 57 | n, stdin, stdout := newNode(t) 58 | kv := maelstrom.NewSeqKV(n) 59 | initNode(t, n, "n1", []string{"n1"}, stdin, stdout) 60 | 61 | errorCh := make(chan error) 62 | go func() { 63 | err := kv.ReadInto(context.Background(), "foo", nil) 64 | if err != nil { 65 | errorCh <- err 66 | return 67 | } 68 | }() 69 | 70 | // Ensure RPC request is received by the network. 71 | if line, err := stdout.ReadString('\n'); err != nil { 72 | t.Fatal(err) 73 | } else if got, want := line, `{"src":"n1","dest":"seq-kv","body":{"key":"foo","msg_id":1,"type":"read"}}`+"\n"; got != want { 74 | t.Fatalf("response=%s, want %s", got, want) 75 | } 76 | 77 | // Write response message back to node. 78 | if _, err := stdin.Write([]byte(`{"src":"seq-kv", "dest":"n1","body":{"type":"read_ok","code":20,"text":"key does not exist","msg_id":2,"in_reply_to":1}}` + "\n")); err != nil { 79 | t.Fatal(err) 80 | } 81 | 82 | // Ensure the response was received. 83 | select { 84 | case err := <-errorCh: 85 | var rpcError *maelstrom.RPCError 86 | if !errors.As(err, &rpcError) { 87 | t.Fatalf("unexpected error type: %#v", err) 88 | } else if got, want := rpcError.Code, 20; got != want { 89 | t.Fatalf("code=%v, want %v", got, want) 90 | } else if got, want := rpcError.Text, "key does not exist"; got != want { 91 | t.Fatalf("text=%v, want %v", got, want) 92 | } 93 | case <-time.After(5 * time.Second): 94 | t.Fatal("timeout waiting for RPC response") 95 | } 96 | }) 97 | } 98 | -------------------------------------------------------------------------------- /demo/go/rpc_error.go: -------------------------------------------------------------------------------- 1 | package maelstrom 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | ) 8 | 9 | // RPC error code constants. 10 | const ( 11 | Timeout = 0 12 | NotSupported = 10 13 | TemporarilyUnavailable = 11 14 | MalformedRequest = 12 15 | Crash = 13 16 | Abort = 14 17 | KeyDoesNotExist = 20 18 | KeyAlreadyExists = 21 19 | PreconditionFailed = 22 20 | TxnConflict = 30 21 | ) 22 | 23 | // ErrorCodeText returns the text representation of an error code. 24 | func ErrorCodeText(code int) string { 25 | switch code { 26 | case Timeout: 27 | return "Timeout" 28 | case NotSupported: 29 | return "NotSupported" 30 | case TemporarilyUnavailable: 31 | return "TemporarilyUnavailable" 32 | case MalformedRequest: 33 | return "MalformedRequest" 34 | case Crash: 35 | return "Crash" 36 | case Abort: 37 | return "Abort" 38 | case KeyDoesNotExist: 39 | return "KeyDoesNotExist" 40 | case KeyAlreadyExists: 41 | return "KeyAlreadyExists" 42 | case PreconditionFailed: 43 | return "PreconditionFailed" 44 | case TxnConflict: 45 | return "TxnConflict" 46 | default: 47 | return fmt.Sprintf("ErrorCode<%d>", code) 48 | } 49 | } 50 | 51 | // ErrorCode returns the error code from err. Returns -1 if err does not have an *RPCError. 52 | func ErrorCode(err error) int { 53 | var rpc *RPCError 54 | if errors.As(err, &rpc) { 55 | return rpc.Code 56 | } 57 | return -1 58 | } 59 | 60 | // RPCError represents a Maelstrom RPC error. 61 | type RPCError struct { 62 | Code int 63 | Text string 64 | } 65 | 66 | // NewRPCError returns a new instance of RPCError. 67 | func NewRPCError(code int, text string) *RPCError { 68 | return &RPCError{ 69 | Code: code, 70 | Text: text, 71 | } 72 | } 73 | 74 | // Error returns a string-formatted error message. 75 | func (e *RPCError) Error() string { 76 | return fmt.Sprintf("RPCError(%s, %q)", ErrorCodeText(e.Code), e.Text) 77 | } 78 | 79 | // MarshalJSON marshals the error into JSON format. 80 | func (e *RPCError) MarshalJSON() ([]byte, error) { 81 | return json.Marshal(rpcErrorJSON{ 82 | Type: "error", 83 | Code: e.Code, 84 | Text: e.Text, 85 | }) 86 | } 87 | 88 | // rpcErrorJSON is a struct for marshaling an RPCError to JSON. 89 | type rpcErrorJSON struct { 90 | Type string `json:"type,omitempty"` 91 | Code int `json:"code,omitempty"` 92 | Text string `json:"text,omitempty"` 93 | } 94 | -------------------------------------------------------------------------------- /demo/go/rpc_error_test.go: -------------------------------------------------------------------------------- 1 | package maelstrom_test 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | maelstrom "github.com/jepsen-io/maelstrom/demo/go" 8 | ) 9 | 10 | func TestErrorCodeText(t *testing.T) { 11 | for _, tt := range []struct { 12 | code int 13 | text string 14 | }{ 15 | {maelstrom.Timeout, "Timeout"}, 16 | {maelstrom.NotSupported, "NotSupported"}, 17 | {maelstrom.TemporarilyUnavailable, "TemporarilyUnavailable"}, 18 | {maelstrom.MalformedRequest, "MalformedRequest"}, 19 | {maelstrom.Crash, "Crash"}, 20 | {maelstrom.Abort, "Abort"}, 21 | {maelstrom.KeyDoesNotExist, "KeyDoesNotExist"}, 22 | {maelstrom.KeyAlreadyExists, "KeyAlreadyExists"}, 23 | {maelstrom.PreconditionFailed, "PreconditionFailed"}, 24 | {maelstrom.TxnConflict, "TxnConflict"}, 25 | {1000, "ErrorCode<1000>"}, 26 | } { 27 | if got, want := maelstrom.ErrorCodeText(tt.code), tt.text; got != want { 28 | t.Errorf("code %d=%s, want %s", tt.code, got, want) 29 | } 30 | } 31 | } 32 | 33 | func TestRPCError_Error(t *testing.T) { 34 | if got, want := maelstrom.NewRPCError(maelstrom.Crash, "foo").Error(), `RPCError(Crash, "foo")`; got != want { 35 | t.Fatalf("error=%s, want %s", got, want) 36 | } 37 | } 38 | 39 | func TestRPCError_ErrorCode(t *testing.T) { 40 | var err error = maelstrom.NewRPCError(maelstrom.Crash, "foo") 41 | if maelstrom.ErrorCode(err) != maelstrom.Crash { 42 | t.Fatalf("error=%d, want %d", maelstrom.ErrorCode(err), maelstrom.Crash) 43 | } 44 | 45 | err = fmt.Errorf("foo: %w", err) 46 | if maelstrom.ErrorCode(err) != maelstrom.Crash { 47 | t.Fatalf("error=%d, want %d", maelstrom.ErrorCode(err), maelstrom.Crash) 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /demo/java/.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | -------------------------------------------------------------------------------- /demo/java/.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "type": "java", 9 | "name": "Launch Current File", 10 | "request": "launch", 11 | "mainClass": "${file}" 12 | }, 13 | { 14 | "type": "java", 15 | "name": "Launch App", 16 | "request": "launch", 17 | "mainClass": "maelstrom.App", 18 | "projectName": "lab" 19 | } 20 | ] 21 | } -------------------------------------------------------------------------------- /demo/java/.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "java.configuration.updateBuildConfiguration": "automatic" 3 | } -------------------------------------------------------------------------------- /demo/java/.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0.0", 3 | "tasks": [ 4 | { 5 | "label": "Maelstrom echo test", 6 | "type": "shell", 7 | "command": "./build && cd ../ && ./maelstrom test --bin lab/server -w echo --time-limit 5 --node-count 1", 8 | "args": [], 9 | "problemMatcher": [ 10 | "$tsc" 11 | ], 12 | "presentation": { 13 | "reveal": "always" 14 | }, 15 | "group": { 16 | "kind": "build", 17 | "isDefault": false 18 | } 19 | }, 20 | { 21 | "label": "Maelstrom broadcast test", 22 | "type": "shell", 23 | "command": "./build && cd ../ && ./maelstrom test --bin lab/server -w broadcast --time-limit 10 --node-count 5", 24 | "args": [], 25 | "problemMatcher": [ 26 | "$tsc" 27 | ], 28 | "presentation": { 29 | "reveal": "always" 30 | }, 31 | "group": { 32 | "kind": "build", 33 | "isDefault": false 34 | } 35 | }, 36 | { 37 | "label": "Maelstrom txn-list-append test", 38 | "type": "shell", 39 | "command": "./build && cd ../../ && ./maelstrom test --bin demo/java/server -w txn-list-append --time-limit 10 --node-count 2 --rate 100", 40 | "args": [], 41 | "problemMatcher": [ 42 | "$tsc" 43 | ], 44 | "presentation": { 45 | "reveal": "always" 46 | }, 47 | "group": { 48 | "kind": "build", 49 | "isDefault": true 50 | } 51 | } 52 | ] 53 | } -------------------------------------------------------------------------------- /demo/java/README.md: -------------------------------------------------------------------------------- 1 | # Maelstrom Java demos 2 | 3 | This Maven project includes a basic echo and broadcast system, as well a 4 | general-purpose node (split into three tiers--Node1, Node2, and Node3, which 5 | provide progressively more sophisticated functionality). The entry point is `maelstrom.Main`. 6 | 7 | Compile this project with `./build`, then in the main `maelstrom` directory, run 8 | 9 | ``` 10 | lein run test -w echo --bin demo/java/server 11 | ``` 12 | 13 | Right now this is hardcoded to be an echo server, and changing that requires 14 | editing the code. Later someone (me?) should go make this a CLI argument, and 15 | maybe add some wrapper scripts. :-) 16 | 17 | There are also Visual Studio Code tasks set up for running Maelstrom tests--see .vscode/tasks.json. 18 | -------------------------------------------------------------------------------- /demo/java/build: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | mvn compile assembly:single 4 | -------------------------------------------------------------------------------- /demo/java/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 4.0.0 5 | 6 | maelstrom 7 | lab 8 | 1.0-SNAPSHOT 9 | 10 | lab 11 | https://jepsen.io 12 | 13 | 14 | UTF-8 15 | 17 16 | 17 17 | 18 | 19 | 20 | 21 | io.lacuna 22 | bifurcan 23 | 0.2.0-alpha4 24 | 25 | 26 | 27 | com.eclipsesource.minimal-json 28 | minimal-json 29 | 0.9.5 30 | 31 | 32 | 33 | junit 34 | junit 35 | 4.13.1 36 | test 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | maven-clean-plugin 47 | 3.1.0 48 | 49 | 50 | 51 | maven-resources-plugin 52 | 3.0.2 53 | 54 | 55 | maven-compiler-plugin 56 | 3.8.0 57 | 58 | 59 | maven-surefire-plugin 60 | 2.22.1 61 | 62 | 73 | 74 | maven-install-plugin 75 | 2.5.2 76 | 77 | 78 | maven-deploy-plugin 79 | 2.8.2 80 | 81 | 82 | 83 | maven-site-plugin 84 | 3.7.1 85 | 86 | 87 | maven-project-info-reports-plugin 88 | 3.0.0 89 | 90 | 91 | org.apache.maven.plugins 92 | maven-assembly-plugin 93 | 3.1.1 94 | 95 | 96 | 97 | jar-with-dependencies 98 | 99 | 100 | 101 | maelstrom.Main 102 | 103 | 104 | 105 | 106 | 107 | 108 | make-assembly 109 | package 110 | 111 | single 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | -------------------------------------------------------------------------------- /demo/java/server: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # http://mywiki.wooledge.org/BashFAQ/028 4 | if [[ $BASH_SOURCE = */* ]]; then 5 | DIR=${BASH_SOURCE%/*}/ 6 | else 7 | DIR=./ 8 | fi 9 | 10 | exec java --enable-preview -Xmx256M -jar "$DIR/target/lab-1.0-SNAPSHOT-jar-with-dependencies.jar" 11 | -------------------------------------------------------------------------------- /demo/java/src/main/java/maelstrom/Error.java: -------------------------------------------------------------------------------- 1 | package maelstrom; 2 | 3 | import com.eclipsesource.json.Json; 4 | import com.eclipsesource.json.JsonValue; 5 | 6 | // The body of a Maelstrom error message, as a throwable exception. 7 | public class Error extends RuntimeException implements IJson { 8 | public final long code; 9 | public final String text; 10 | 11 | public Error(long code, String text) { 12 | this.code = code; 13 | this.text = text; 14 | } 15 | 16 | // Utility functions to construct specific kinds of errors, following https://github.com/jepsen-io/maelstrom/blob/main/doc/protocol.md#errors 17 | public static Error timeout(String text) { 18 | return new Error(0, text); 19 | } 20 | public static Error notSupported(String text) { 21 | return new Error(10, text); 22 | } 23 | public static Error temporarilyUnavailable(String text) { 24 | return new Error(11, text); 25 | } 26 | public static Error malformedRequest(String text) { 27 | return new Error(12, text); 28 | } 29 | public static Error crash(String text) { 30 | return new Error(13, text); 31 | } 32 | public static Error abort(String text) { 33 | return new Error(14, text); 34 | } 35 | public static Error keyDoesNotExist(String text) { 36 | return new Error(20, text); 37 | } 38 | public static Error keyAlreadyExists(String text) { 39 | return new Error(21, text); 40 | } 41 | public static Error preconditionFailed(String text) { 42 | return new Error(22, text); 43 | } 44 | public static Error txnConflict(String text) { 45 | return new Error(30, text); 46 | } 47 | 48 | public JsonValue toJson() { 49 | return Json.object() 50 | .add("type", "error") 51 | .add("code", code) 52 | .add("text", text); 53 | } 54 | 55 | public String toString() { 56 | return toJson().toString(); 57 | } 58 | } -------------------------------------------------------------------------------- /demo/java/src/main/java/maelstrom/IJson.java: -------------------------------------------------------------------------------- 1 | package maelstrom; 2 | 3 | import com.eclipsesource.json.JsonValue; 4 | 5 | // Support for coercing datatypes to and from JsonValues. 6 | public interface IJson { 7 | // Coerce something to a JsonValue. 8 | public JsonValue toJson(); 9 | } -------------------------------------------------------------------------------- /demo/java/src/main/java/maelstrom/JsonUtil.java: -------------------------------------------------------------------------------- 1 | package maelstrom; 2 | 3 | import java.util.function.Function; 4 | 5 | import com.eclipsesource.json.Json; 6 | import com.eclipsesource.json.JsonArray; 7 | import com.eclipsesource.json.JsonObject; 8 | import com.eclipsesource.json.JsonValue; 9 | import com.eclipsesource.json.JsonObject.Member; 10 | 11 | import io.lacuna.bifurcan.IMap; 12 | import io.lacuna.bifurcan.List; 13 | import io.lacuna.bifurcan.Map; 14 | 15 | // A few basic coercions for JSON 16 | public class JsonUtil { 17 | // Functions to coerce JsonValues to longs, strings, etc. 18 | public static Function asLong = (j) -> { return j.asLong(); }; 19 | public static Function asString = (j) -> { return j.asString(); }; 20 | 21 | // Convert an iterable of longs to a JsonArray 22 | public static JsonArray longsToJson(Iterable iterable) { 23 | final JsonArray a = Json.array(); 24 | for (long e : iterable) { 25 | a.add(e); 26 | } 27 | return a; 28 | } 29 | 30 | // Convert an iterable of T to a JsonArray via a function. 31 | public static JsonArray toJson(Iterable iterable, Function f) { 32 | final JsonArray a = Json.array(); 33 | for (T e : iterable) { 34 | a.add(f.apply(e)); 35 | } 36 | return a; 37 | } 38 | 39 | // Convert an iterable of IJsons to a JsonArray 40 | public static JsonArray toJson(Iterable iterable) { 41 | final JsonArray a = Json.array(); 42 | for (IJson e : iterable) { 43 | a.add(e.toJson()); 44 | } 45 | return a; 46 | } 47 | 48 | // Convert a json array to a bifurcan list, transforming elements using the given function. 49 | public static List toBifurcanList(JsonArray a, Function f) { 50 | List list = new List().forked(); 51 | for (JsonValue element : a) { 52 | list = list.addLast(f.apply(element)); 53 | } 54 | return list.forked(); 55 | } 56 | 57 | // Convert a json object to a bifurcan map, transforming keys and values using 58 | // the given functions. 59 | public static IMap toBifurcanMap(JsonObject o, Function keyFn, Function valFn) { 60 | IMap m = new Map().linear(); 61 | for (Member pair : o) { 62 | m = m.put(keyFn.apply(pair.getName()), 63 | valFn.apply(pair.getValue())); 64 | } 65 | return m.forked(); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /demo/java/src/main/java/maelstrom/Main.java: -------------------------------------------------------------------------------- 1 | package maelstrom; 2 | 3 | import maelstrom.txnListAppend.TxnListAppendServer; 4 | 5 | public class Main { 6 | public static void main(String[] args) { 7 | // new EchoServer().run(); 8 | // new BroadcastServer().run(); 9 | new TxnListAppendServer().run(); 10 | } 11 | } -------------------------------------------------------------------------------- /demo/java/src/main/java/maelstrom/Message.java: -------------------------------------------------------------------------------- 1 | package maelstrom; 2 | 3 | import com.eclipsesource.json.Json; 4 | import com.eclipsesource.json.JsonObject; 5 | import com.eclipsesource.json.JsonValue; 6 | 7 | // A message contains a source and destination, and a JsonObject body. 8 | public class Message implements IJson { 9 | public final String src; 10 | public final String dest; 11 | public final JsonObject body; 12 | 13 | public Message(String src, String dest, JsonObject body) { 14 | this.src = src; 15 | this.dest = dest; 16 | this.body = body; 17 | } 18 | 19 | public Message(String src, String dest, IJson body) { 20 | this(src, dest, body.toJson().asObject()); 21 | } 22 | 23 | // Parse a JsonObject as a message. 24 | public Message(JsonValue j) { 25 | final JsonObject o = j.asObject(); 26 | this.src = o.getString("src", null); 27 | this.dest = o.getString("dest", null); 28 | this.body = o.get("body").asObject(); 29 | } 30 | 31 | public String toString() { 32 | return "(msg " + src + " " + dest + " " + body + ")"; 33 | } 34 | 35 | @Override 36 | public JsonValue toJson() { 37 | return Json.object() 38 | .add("src", src) 39 | .add("dest", dest) 40 | .add("body", body); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /demo/java/src/main/java/maelstrom/Node1.java: -------------------------------------------------------------------------------- 1 | package maelstrom; 2 | 3 | import java.text.DateFormat; 4 | import java.text.SimpleDateFormat; 5 | import java.util.ArrayList; 6 | import java.util.Date; 7 | import java.util.HashMap; 8 | import java.util.List; 9 | import java.util.Map; 10 | import java.util.Scanner; 11 | import java.util.TimeZone; 12 | import java.util.function.Consumer; 13 | 14 | import com.eclipsesource.json.Json; 15 | import com.eclipsesource.json.JsonObject; 16 | import com.eclipsesource.json.JsonValue; 17 | 18 | // A minimal Maelstrom node, with a basic mainloop, support for init messages, and pluggable RPC request handlers 19 | public class Node1 { 20 | // Our local node ID. 21 | public String nodeId = "uninitialized"; 22 | 23 | // All node IDs 24 | public List nodeIds = new ArrayList(); 25 | 26 | // A map of request RPC types (e.g. "echo") to Consumers which should 27 | // be invoked when those messages arrive. 28 | public final Map> requestHandlers = new HashMap>(); 29 | 30 | public Node1() { 31 | } 32 | 33 | // Registers a request handler for the given type of message. 34 | public Node1 on(String type, Consumer handler) { 35 | requestHandlers.put(type, handler); 36 | return this; 37 | } 38 | 39 | // Log a message to stderr. 40 | public void log(String message) { 41 | TimeZone tz = TimeZone.getDefault(); 42 | DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); 43 | df.setTimeZone(tz); 44 | System.err.println(df.format(new Date()) + " " + message); 45 | System.err.flush(); 46 | } 47 | 48 | // Sending messages ////////////////////////////////////////////////////// 49 | 50 | // Send a message to stdout 51 | public void send(final Message message) { 52 | log("Sending " + message.toJson()); 53 | System.out.println(message.toJson()); 54 | System.out.flush(); 55 | } 56 | 57 | // Send a message to a specific node. 58 | public void send(String dest, JsonObject body) { 59 | send(new Message(nodeId, dest, body)); 60 | } 61 | 62 | // Reply to a specific request message with a JsonObject body. 63 | public void reply(Message request, JsonObject body) { 64 | final Long msg_id = request.body.getLong("msg_id", -1); 65 | final JsonObject body2 = Json.object().merge(body).set("in_reply_to", msg_id); 66 | send(request.src, body2); 67 | } 68 | 69 | // Handlers //////////////////////////////////////////////////////////// 70 | 71 | // Handle an init message, setting up our state. 72 | public void handleInit(Message request) { 73 | this.nodeId = request.body.getString("node_id", null); 74 | for (JsonValue id : request.body.get("node_ids").asArray()) { 75 | this.nodeIds.add(id.asString()); 76 | } 77 | log("I am " + nodeId); 78 | } 79 | 80 | // Handle a message by looking up a request handler by the type of the message's 81 | // body, and calling it with the message. 82 | public void handleRequest(Message request) { 83 | final String type = request.body.getString("type", null); 84 | Consumer handler = requestHandlers.get(type); 85 | if (handler == null) { 86 | // You don't have to register a custom init handler. 87 | if (type.equals("init")) { 88 | return; 89 | } 90 | throw Error.notSupported("Don't know how to handle a request of type " + type); 91 | } 92 | handler.accept(request); 93 | } 94 | 95 | // Handles a parsed message from STDIN 96 | public void handleMessage(Message message) { 97 | final JsonObject body = message.body; 98 | final String type = body.getString("type", null); 99 | log("Handling " + message); 100 | 101 | // Init messages are special: we always handle them ourselves in addition to 102 | // invoking any registered callback. 103 | if (type.equals("init")) { 104 | handleInit(message); 105 | handleRequest(message); 106 | reply(message, Json.object().add("type", "init_ok")); 107 | } else { 108 | // Dispatch based on message type. 109 | handleRequest(message); 110 | } 111 | } 112 | 113 | // The mainloop. Consumes lines of JSON from STDIN. Invoke this once the node is 114 | // configured to begin handling requests. 115 | public void main() { 116 | final Scanner scanner = new Scanner(System.in); 117 | String line; 118 | Message message; 119 | try { 120 | while (true) { 121 | line = scanner.nextLine(); 122 | message = new Message(Json.parse(line).asObject()); 123 | handleMessage(message); 124 | } 125 | } catch (Throwable e) { 126 | log("Fatal error! " + e); 127 | e.printStackTrace(); 128 | System.exit(1); 129 | } finally { 130 | scanner.close(); 131 | } 132 | } 133 | } -------------------------------------------------------------------------------- /demo/java/src/main/java/maelstrom/Node2.java: -------------------------------------------------------------------------------- 1 | package maelstrom; 2 | 3 | import java.io.PrintWriter; 4 | import java.io.StringWriter; 5 | import java.text.DateFormat; 6 | import java.text.SimpleDateFormat; 7 | import java.util.ArrayList; 8 | import java.util.Date; 9 | import java.util.HashMap; 10 | import java.util.List; 11 | import java.util.Map; 12 | import java.util.Scanner; 13 | import java.util.TimeZone; 14 | import java.util.function.Consumer; 15 | 16 | import com.eclipsesource.json.Json; 17 | import com.eclipsesource.json.JsonObject; 18 | import com.eclipsesource.json.JsonValue; 19 | 20 | // This class provides common support functions for writing Maelstrom nodes, and includes automatic error handling and broader support for issuing messages to other nodes. 21 | public class Node2 { 22 | // Our local node ID. 23 | public String nodeId = "uninitialized"; 24 | 25 | // All node IDs 26 | public List nodeIds = new ArrayList(); 27 | 28 | // A map of request RPC types (e.g. "echo") to Consumers which should 29 | // be invoked when those messages arrive. 30 | public final Map> requestHandlers = new HashMap>(); 31 | 32 | public Node2() { 33 | } 34 | 35 | // Registers a request handler for the given type of message. 36 | public Node2 on(String type, Consumer handler) { 37 | requestHandlers.put(type, handler); 38 | return this; 39 | } 40 | 41 | // Log a message to stderr. 42 | public void log(String message) { 43 | TimeZone tz = TimeZone.getDefault(); 44 | DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); 45 | df.setTimeZone(tz); 46 | System.err.println(df.format(new Date()) + " " + message); 47 | System.err.flush(); 48 | } 49 | 50 | // Sending messages ////////////////////////////////////////////////////// 51 | 52 | // Send a message to stdout 53 | public void send(final Message message) { 54 | log("Sending " + message.toJson()); 55 | System.out.println(message.toJson()); 56 | System.out.flush(); 57 | } 58 | 59 | // Send a message to a specific node. 60 | public void send(String dest, JsonObject body) { 61 | send(new Message(nodeId, dest, body)); 62 | } 63 | 64 | // Reply to a specific request message with a JsonObject body. 65 | public void reply(Message request, JsonObject body) { 66 | final Long msg_id = request.body.getLong("msg_id", -1); 67 | final JsonObject body2 = Json.object().merge(body).set("in_reply_to", msg_id); 68 | send(request.src, body2); 69 | } 70 | 71 | // Reply to a message with a Json-coercable object as the body. 72 | public void reply(Message request, IJson body) { 73 | reply(request, body.toJson().asObject()); 74 | } 75 | 76 | // Handlers //////////////////////////////////////////////////////////// 77 | 78 | // Handle an init message, setting up our state. 79 | public void handleInit(Message request) { 80 | this.nodeId = request.body.getString("node_id", null); 81 | for (JsonValue id : request.body.get("node_ids").asArray()) { 82 | this.nodeIds.add(id.asString()); 83 | } 84 | log("I am " + nodeId); 85 | } 86 | 87 | // Handle a message by looking up a request handler by the type of the message's 88 | // body, and calling it with the message. 89 | public void handleRequest(Message request) { 90 | final String type = request.body.getString("type", null); 91 | Consumer handler = requestHandlers.get(type); 92 | if (handler == null) { 93 | // You don't have to register a custom init handler. 94 | if (type.equals("init")) { 95 | return; 96 | } 97 | throw Error.notSupported("Don't know how to handle a request of type " + type); 98 | } 99 | handler.accept(request); 100 | } 101 | 102 | // Handles a parsed message from STDIN 103 | public void handleMessage(Message message) { 104 | final JsonObject body = message.body; 105 | final String type = body.getString("type", null); 106 | log("Handling " + message); 107 | 108 | try { 109 | // Init messages are special: we always handle them ourselves in addition to 110 | // invoking any registered callback. 111 | if (type.equals("init")) { 112 | handleInit(message); 113 | handleRequest(message); 114 | reply(message, Json.object().add("type", "init_ok")); 115 | } else { 116 | // Dispatch based on message type. 117 | handleRequest(message); 118 | } 119 | } catch (Error e) { 120 | // Send a message back to the client 121 | log(e.toString()); 122 | reply(message, e); 123 | } catch (Exception e) { 124 | // Send a generic crash error 125 | StringWriter sw = new StringWriter(); 126 | PrintWriter pw = new PrintWriter(sw); 127 | e.printStackTrace(pw); 128 | String text = "Unexpected exception handling " + 129 | message + ": " + e + "\n" + sw; 130 | log(text); 131 | reply(message, Error.crash(text)); 132 | } 133 | } 134 | 135 | // The mainloop. Consumes lines of JSON from STDIN. Invoke this once the node is 136 | // configured to begin handling requests. 137 | public void main() { 138 | final Scanner scanner = new Scanner(System.in); 139 | String line; 140 | Message message; 141 | try { 142 | while (true) { 143 | line = scanner.nextLine(); 144 | message = new Message(Json.parse(line).asObject()); 145 | handleMessage(message); 146 | } 147 | } catch (Throwable e) { 148 | log("Fatal error! " + e); 149 | e.printStackTrace(); 150 | System.exit(1); 151 | } finally { 152 | scanner.close(); 153 | } 154 | } 155 | } -------------------------------------------------------------------------------- /demo/java/src/main/java/maelstrom/broadcast/BroadcastServer.java: -------------------------------------------------------------------------------- 1 | package maelstrom.broadcast; 2 | 3 | import java.util.ArrayList; 4 | import java.util.HashSet; 5 | import java.util.List; 6 | import java.util.Set; 7 | import java.util.concurrent.TimeUnit; 8 | 9 | import com.eclipsesource.json.Json; 10 | import com.eclipsesource.json.JsonArray; 11 | import com.eclipsesource.json.JsonObject; 12 | import com.eclipsesource.json.JsonValue; 13 | 14 | import maelstrom.JsonUtil; 15 | import maelstrom.Node3; 16 | 17 | public class BroadcastServer { 18 | public final Node3 node = new Node3(); 19 | // The nodes we're directly adjacent to 20 | public List neighbors = new ArrayList(); 21 | // All messages we know about 22 | public Set messages = new HashSet(); 23 | 24 | // Sends a message to a neighbor and keep retrying if it times out or fails 25 | public void broadcastToNeighbor(String neighbor, JsonObject message) { 26 | node.rpc(neighbor, message) 27 | .orTimeout(1000, TimeUnit.MILLISECONDS) 28 | .exceptionally((t) -> { 29 | node.log("Retrying broadcast of " + message + " to " + neighbor); 30 | broadcastToNeighbor(neighbor, message); 31 | return null; 32 | }); 33 | } 34 | 35 | public void run() { 36 | // When we get a topology message, record our neighbors 37 | node.on("topology", (req) -> { 38 | final JsonArray neighbors = req.body 39 | .get("topology").asObject() 40 | .get(node.nodeId).asArray(); 41 | for (JsonValue neighbor : neighbors) { 42 | this.neighbors.add(neighbor.asString()); 43 | } 44 | node.reply(req, Json.object().add("type", "topology_ok")); 45 | }); 46 | 47 | // When we get a read, respond with our set of messages 48 | node.on("read", (req) -> { 49 | node.reply(req, Json.object() 50 | .add("type", "read_ok") 51 | .add("messages", JsonUtil.longsToJson(messages))); 52 | }); 53 | 54 | // And when we get an add, add it to the local set and broadcast it out 55 | node.on("broadcast", (req) -> { 56 | final Long message = req.body.getLong("message", -1); 57 | if (!messages.contains(message)) { 58 | messages.add(message); 59 | for (String neighbor : neighbors) { 60 | broadcastToNeighbor(neighbor, 61 | Json.object() 62 | .add("type", "broadcast") 63 | .add("message", message)); 64 | } 65 | } 66 | 67 | node.reply(req, Json.object().add("type", "broadcast_ok")); 68 | }); 69 | 70 | node.log("Starting up!"); 71 | node.main(); 72 | } 73 | } -------------------------------------------------------------------------------- /demo/java/src/main/java/maelstrom/echo/EchoServer.java: -------------------------------------------------------------------------------- 1 | package maelstrom.echo; 2 | 3 | import com.eclipsesource.json.Json; 4 | 5 | import maelstrom.Node1; 6 | 7 | public class EchoServer { 8 | public void run() { 9 | final Node1 node = new Node1(); 10 | node.on("echo", (req) -> { 11 | node.reply(req, Json.object() 12 | .add("type", "echo_ok") 13 | .add("echo", req.body.getString("echo", null))); 14 | }); 15 | node.log("Starting up!"); 16 | node.main(); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /demo/java/src/main/java/maelstrom/txnListAppend/AppendOp.java: -------------------------------------------------------------------------------- 1 | package maelstrom.txnListAppend; 2 | 3 | import com.eclipsesource.json.Json; 4 | import com.eclipsesource.json.JsonArray; 5 | import com.eclipsesource.json.JsonValue; 6 | 7 | // An append operation is a triple of a function, a key, and a long value. 8 | public record AppendOp(String fun, Long key, Long value) implements IOp { 9 | public static AppendOp from(JsonArray json) { 10 | return new AppendOp(json.get(0).asString(), 11 | json.get(1).asLong(), 12 | json.get(2).asLong()); 13 | } 14 | 15 | @Override 16 | public String getFun() { 17 | return fun; 18 | } 19 | 20 | @Override 21 | public Long getKey() { 22 | return key; 23 | } 24 | 25 | @Override 26 | public Object getValue() { 27 | return value; 28 | } 29 | 30 | @Override 31 | public JsonValue toJson() { 32 | return Json.array().add(fun).add(key).add(value); 33 | } 34 | } -------------------------------------------------------------------------------- /demo/java/src/main/java/maelstrom/txnListAppend/IOp.java: -------------------------------------------------------------------------------- 1 | package maelstrom.txnListAppend; 2 | 3 | import com.eclipsesource.json.JsonArray; 4 | 5 | import maelstrom.IJson; 6 | 7 | // Represents a tranasction operation 8 | public interface IOp extends IJson { 9 | public String getFun(); 10 | 11 | public Long getKey(); 12 | 13 | public Object getValue(); 14 | 15 | // Constructs either an append or read op from a JSON array. 16 | public static IOp from(JsonArray json) { 17 | final String fun = json.get(0).asString(); 18 | switch (fun) { 19 | case "append": 20 | return AppendOp.from(json); 21 | case "r": 22 | return ReadOp.from(json); 23 | default: 24 | throw new IllegalArgumentException("Unknown op fun: " + fun); 25 | } 26 | } 27 | } -------------------------------------------------------------------------------- /demo/java/src/main/java/maelstrom/txnListAppend/KVStore.java: -------------------------------------------------------------------------------- 1 | package maelstrom.txnListAppend; 2 | 3 | import java.util.concurrent.CompletableFuture; 4 | import java.util.concurrent.CompletionException; 5 | 6 | import com.eclipsesource.json.Json; 7 | import com.eclipsesource.json.JsonObject; 8 | import com.eclipsesource.json.JsonValue; 9 | import maelstrom.Error; 10 | import maelstrom.Node3; 11 | 12 | // A KVStore represents a Maelstrom KV service. 13 | public class KVStore { 14 | final Node3 node; 15 | final String service; 16 | 17 | public KVStore(Node3 node, String service) { 18 | this.node = node; 19 | this.service = service; 20 | } 21 | 22 | // Reads the given key 23 | public CompletableFuture read(String k) { 24 | return node.rpc(service, 25 | Json.object() 26 | .add("type", "read") 27 | .add("key", k)) 28 | .thenApply((res) -> { 29 | return res.get("value"); 30 | }); 31 | } 32 | 33 | // Reads the given key, returning a default if not found 34 | public CompletableFuture read(String k, JsonValue defaultValue) { 35 | return read(k).exceptionally((e) -> { 36 | // Unwrap completionexceptions 37 | if (e instanceof CompletionException) { 38 | e = e.getCause(); 39 | } 40 | if (e instanceof Error) { 41 | final Error err = (Error) e; 42 | if (err.code == 20) { 43 | return defaultValue; 44 | } 45 | } 46 | throw new RuntimeException(e); 47 | }); 48 | } 49 | 50 | // Reads the given key, retrying not-found errors 51 | public CompletableFuture readUntilFound(String k) { 52 | return read(k).exceptionally((e) -> { 53 | if (e instanceof CompletionException) { 54 | e = e.getCause(); 55 | } 56 | if (e instanceof Error) { 57 | final Error err = (Error) e; 58 | if (err.code == 20) { 59 | return readUntilFound(k).join(); 60 | } 61 | } 62 | throw new RuntimeException(e); 63 | }); 64 | } 65 | 66 | // Writes the given key 67 | public CompletableFuture write(String k, JsonValue v) { 68 | return node.rpc(service, Json.object() 69 | .add("type", "write") 70 | .add("key", k) 71 | .add("value", v)); 72 | } 73 | 74 | // Compare-and-sets the given key from `from` to `to`. Creates key if it doesn't 75 | // exist. 76 | public CompletableFuture cas(String k, JsonValue from, JsonValue to) { 77 | return node.rpc(service, Json.object() 78 | .add("type", "cas") 79 | .add("key", k) 80 | .add("from", from) 81 | .add("to", to) 82 | .add("create_if_not_exists", true)); 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /demo/java/src/main/java/maelstrom/txnListAppend/ReadOp.java: -------------------------------------------------------------------------------- 1 | package maelstrom.txnListAppend; 2 | 3 | import com.eclipsesource.json.Json; 4 | import com.eclipsesource.json.JsonArray; 5 | import com.eclipsesource.json.JsonValue; 6 | 7 | import io.lacuna.bifurcan.List; 8 | import maelstrom.JsonUtil; 9 | 10 | // A read operation is a triple of a function, a key, and a list of longs. 11 | public record ReadOp(String fun, Long key, List value) implements IOp { 12 | public static ReadOp from(JsonArray json) { 13 | final JsonValue jv = json.get(2); 14 | List value = null; 15 | if (! jv.isNull()) { 16 | value = JsonUtil.toBifurcanList(jv.asArray(), JsonUtil.asLong); 17 | } 18 | return new ReadOp(json.get(0).asString(), 19 | json.get(1).asLong(), 20 | value); 21 | } 22 | 23 | @Override 24 | public String getFun() { 25 | return fun; 26 | } 27 | 28 | @Override 29 | public Long getKey() { 30 | return key; 31 | } 32 | 33 | @Override 34 | public Object getValue() { 35 | return value; 36 | } 37 | 38 | @Override 39 | public JsonValue toJson() { 40 | final JsonArray a = Json.array().add(fun).add(key); 41 | if (value == null) { 42 | a.add(Json.NULL); 43 | } else { 44 | a.add(JsonUtil.longsToJson(value)); 45 | } 46 | return a; 47 | }; 48 | } -------------------------------------------------------------------------------- /demo/java/src/main/java/maelstrom/txnListAppend/Root.java: -------------------------------------------------------------------------------- 1 | package maelstrom.txnListAppend; 2 | 3 | import com.eclipsesource.json.Json; 4 | import com.eclipsesource.json.JsonArray; 5 | import com.eclipsesource.json.JsonValue; 6 | 7 | import io.lacuna.bifurcan.IEntry; 8 | import io.lacuna.bifurcan.IMap; 9 | import io.lacuna.bifurcan.SortedMap; 10 | import maelstrom.IJson; 11 | 12 | // A Root is a map of keys to thunk IDs. 13 | public record Root(IMap map) implements IJson { 14 | // An empty root 15 | public Root() { 16 | this(new SortedMap().forked()); 17 | } 18 | 19 | // Inflate a root from a JSON array 20 | public static Root from(JsonValue json) { 21 | final JsonArray a = json.asArray(); 22 | IMap m = new SortedMap().linear(); 23 | for (int i = 0; i < a.size(); i = i + 2) { 24 | m = m.put(a.get(i).asLong(), a.get(i + 1).asString()); 25 | } 26 | return new Root(m.forked()); 27 | } 28 | 29 | // We serialize roots as a list of flat [k v k v] pairs. 30 | public JsonValue toJson() { 31 | JsonArray a = Json.array(); 32 | for (IEntry pair: map) { 33 | a = a.add(pair.key()).add(pair.value()); 34 | } 35 | return a; 36 | } 37 | 38 | // Merge two roots together 39 | public Root merge(Root other) { 40 | return new Root(map.merge(other.map(), (a, b) -> b)); 41 | } 42 | } -------------------------------------------------------------------------------- /demo/java/src/main/java/maelstrom/txnListAppend/State.java: -------------------------------------------------------------------------------- 1 | package maelstrom.txnListAppend; 2 | 3 | import io.lacuna.bifurcan.IMap; 4 | import io.lacuna.bifurcan.List; 5 | 6 | // A State is a map of longs to lists of longs. 7 | public record State(IMap> map) { 8 | // A pair of a state and transaction together. 9 | public record StateTxn(State state, Txn txn) { 10 | } 11 | 12 | // Applies a transaction to a state, returning a new State and completed Txn. 13 | public StateTxn apply(Txn txn) { 14 | // A mutable copy of the transaction operations, which we'll fill in with 15 | // completed ops as we go. 16 | List ops2 = new List().linear(); 17 | // A mutable copy of our state map, which we'll evolve as we go. 18 | IMap> map2 = map.linear(); 19 | 20 | for (final IOp op : txn.ops()) { 21 | final String f = op.getFun(); 22 | final Long k = op.getKey(); 23 | final Object v = op.getValue(); 24 | switch (f) { 25 | case "append": 26 | map2 = map2.put(k, map2.get(op.getKey(), new List()).addLast((Long) v)); 27 | ops2 = ops2.addLast(op); 28 | break; 29 | 30 | case "r": 31 | ops2 = ops2.addLast(new ReadOp(f, k, map2.get(k, null))); 32 | break; 33 | 34 | default: 35 | throw new IllegalArgumentException("Unexpected op fun " + f); 36 | } 37 | } 38 | 39 | return new StateTxn(new State(map2.forked()), new Txn(ops2.forked())); 40 | } 41 | } -------------------------------------------------------------------------------- /demo/java/src/main/java/maelstrom/txnListAppend/Thunk.java: -------------------------------------------------------------------------------- 1 | package maelstrom.txnListAppend; 2 | 3 | import io.lacuna.bifurcan.List; 4 | 5 | // A thunk stores a list of numbers under a unique string ID. 6 | public record Thunk(String id, List value) { 7 | } -------------------------------------------------------------------------------- /demo/java/src/main/java/maelstrom/txnListAppend/Txn.java: -------------------------------------------------------------------------------- 1 | package maelstrom.txnListAppend; 2 | 3 | import com.eclipsesource.json.JsonArray; 4 | import com.eclipsesource.json.JsonValue; 5 | 6 | import io.lacuna.bifurcan.List; 7 | import io.lacuna.bifurcan.Set; 8 | import maelstrom.IJson; 9 | import maelstrom.JsonUtil; 10 | 11 | // A transaction is a list of operations. 12 | public record Txn(List ops) implements IJson { 13 | // Build a transaction from a JSON array 14 | public Txn(JsonArray json) { 15 | this(JsonUtil.toBifurcanList(json, (op) -> { 16 | return IOp.from(op.asArray()); 17 | })); 18 | } 19 | 20 | @Override 21 | public JsonValue toJson() { 22 | return JsonUtil.toJson(ops, (op) -> { 23 | return op.toJson(); 24 | }); 25 | } 26 | 27 | // The set of all keys we need to read in order to execute a transaction. 28 | public final Set readSet() { 29 | Set keys = new Set().linear(); 30 | for (IOp op : ops) { 31 | keys.add(op.getKey()); 32 | } 33 | return keys.forked(); 34 | } 35 | 36 | // The set of all keys we need to write in order to execute a transaction. 37 | public final Set writeSet() { 38 | Set keys = new Set().linear(); 39 | for (IOp op : ops) { 40 | if (! (op instanceof ReadOp)) { 41 | keys.add(op.getKey()); 42 | } 43 | } 44 | return keys.forked(); 45 | } 46 | } -------------------------------------------------------------------------------- /demo/java/src/main/java/maelstrom/txnListAppend/TxnListAppendServer.java: -------------------------------------------------------------------------------- 1 | package maelstrom.txnListAppend; 2 | 3 | import java.util.concurrent.CompletableFuture; 4 | import java.util.concurrent.CompletionException; 5 | import java.util.concurrent.ConcurrentHashMap; 6 | import java.util.concurrent.atomic.AtomicLong; 7 | import java.util.concurrent.atomic.AtomicReference; 8 | 9 | import com.eclipsesource.json.Json; 10 | import com.eclipsesource.json.JsonObject; 11 | 12 | import io.lacuna.bifurcan.IMap; 13 | import io.lacuna.bifurcan.List; 14 | import io.lacuna.bifurcan.Map; 15 | import io.lacuna.bifurcan.Set; 16 | import maelstrom.Error; 17 | import maelstrom.JsonUtil; 18 | import maelstrom.Node3; 19 | import maelstrom.txnListAppend.State.StateTxn; 20 | 21 | public class TxnListAppendServer { 22 | public final Node3 node = new Node3(); 23 | public final KVStore thunkStore = new KVStore(node, "lin-kv"); 24 | public final KVStore rootStore = new KVStore(node, "lin-kv"); 25 | // Where do we store the root? 26 | public final String rootKey = "root"; 27 | 28 | // What's the next thunk number? 29 | public static final AtomicLong nextThunkNumber = new AtomicLong(0); 30 | 31 | // A mutable cache of thunk IDs to thunks 32 | public static final ConcurrentHashMap thunkCache = new ConcurrentHashMap(); 33 | 34 | // A cache for what we think the current root is 35 | public final AtomicReference rootCache = new AtomicReference<>(new Root()); 36 | 37 | // Generate a new thunk ID for a node. 38 | public final String newThunkId() { 39 | return node.nodeId + "-" + nextThunkNumber.getAndIncrement(); 40 | } 41 | 42 | // Fetch a thunk, either from cache or storage 43 | public CompletableFuture loadThunk(String id) { 44 | // Try cache 45 | if (thunkCache.containsKey(id)) { 46 | node.log("cache hit " + id); 47 | final CompletableFuture f = new CompletableFuture(); 48 | f.complete(thunkCache.get(id)); 49 | return f; 50 | } 51 | 52 | // Fall back to KV store 53 | return thunkStore.readUntilFound(id).thenApply((json) -> { 54 | final Thunk thunk = new Thunk(id, JsonUtil.toBifurcanList(json.asArray(), (element) -> { 55 | return element.asLong(); 56 | })); 57 | // Update cache 58 | thunkCache.put(id, thunk); 59 | return thunk; 60 | }); 61 | } 62 | 63 | // Write a thunk to storage and cache 64 | public CompletableFuture saveThunk(Thunk thunk) { 65 | thunkCache.put(thunk.id(), thunk); 66 | return thunkStore.write(thunk.id(), JsonUtil.longsToJson(thunk.value())); 67 | } 68 | 69 | // Takes a Root and a set of keys. Returns a (partial) State constructed by 70 | // looking up those keys' corresponding thunks in the root. 71 | public State loadPartialState(Root root, Set keys) { 72 | final IMap rootMap = root.map(); 73 | final ConcurrentHashMap> stateMap = new ConcurrentHashMap<>(); 74 | // Fire off reads for each key with a thunk 75 | keys.stream().filter((k) -> { 76 | return rootMap.contains(k); 77 | }).map((k) -> { 78 | final String thunkId = rootMap.get(k, null); 79 | return loadThunk(thunkId).thenAccept((thunk) -> { 80 | stateMap.put(k, thunk.value()); 81 | }); 82 | }).toList().stream().forEach((future) -> { 83 | // And block on each read 84 | future.join(); 85 | }); 86 | return new State(Map.from(stateMap)); 87 | } 88 | 89 | // Takes a State and a set of keys. Generates a fresh thunk ID for each of those 90 | // keys, and writes the corresponding value to the thunk store. Returns a 91 | // (partial) Root which maps those keys to their newly-created thunk IDs. 92 | public Root savePartialState(State state, Set keys) { 93 | final IMap> stateMap = state.map(); 94 | final ConcurrentHashMap rootMap = new ConcurrentHashMap<>(); 95 | // Fire off writes for each key 96 | keys.stream().map(k -> { 97 | final String thunkId = newThunkId(); 98 | final Thunk thunk = new Thunk(thunkId, stateMap.get(k, null)); 99 | rootMap.put(k, thunkId); 100 | return saveThunk(thunk); 101 | }).toList().stream().forEach((future) -> { 102 | // Block on each write 103 | future.join(); 104 | }); 105 | return new Root(Map.from(rootMap)); 106 | } 107 | 108 | public void run() { 109 | node.on("txn", (req) -> { 110 | // Parse txn 111 | final Txn txn = new Txn(req.body.get("txn").asArray()); 112 | // Use cached root 113 | final Root root = rootCache.get(); 114 | // Fetch thunks and construct a partial state 115 | final State state = loadPartialState(root, txn.readSet()); 116 | // Apply txn 117 | final StateTxn stateTxn2 = state.apply(txn); 118 | final State state2 = stateTxn2.state(); 119 | final Txn txn2 = stateTxn2.txn(); 120 | // Save thunks and merge into root 121 | final Root root2 = root.merge(savePartialState(state2, txn.writeSet())); 122 | // Update root 123 | try { 124 | rootStore.cas(rootKey, root.toJson(), root2.toJson()).join(); 125 | rootCache.compareAndSet(root, root2); 126 | // And confirm txn 127 | node.reply(req, Json.object() 128 | .add("type", "txn_ok") 129 | .add("txn", txn2.toJson())); 130 | } catch (CompletionException e) { 131 | if (e.getCause() instanceof Error) { 132 | final Error err = (Error) e.getCause(); 133 | if (err.code == 22) { 134 | // Cas failed because of mismatching precondition. Reload root! 135 | rootStore.read(rootKey, Json.array()).thenAccept((rootJson) -> { 136 | node.log("Refreshed root"); 137 | rootCache.set(Root.from(rootJson)); 138 | }); 139 | throw Error.txnConflict("root altered"); 140 | } else { 141 | // Some other Maelstrom error 142 | throw Error.abort("Unexpected error updating root: " + err); 143 | } 144 | } else { 145 | throw e; 146 | } 147 | } 148 | }); 149 | 150 | node.log("Starting up!"); 151 | node.main(); 152 | } 153 | } -------------------------------------------------------------------------------- /demo/java/src/test/java/maelstrom/AppTest.java: -------------------------------------------------------------------------------- 1 | package maelstrom; 2 | 3 | import static org.junit.Assert.assertTrue; 4 | 5 | import org.junit.Test; 6 | 7 | /** 8 | * Unit test for simple App. 9 | */ 10 | public class AppTest 11 | { 12 | /** 13 | * Rigorous Test :-) 14 | */ 15 | @Test 16 | public void shouldAnswerWithTrue() 17 | { 18 | assertTrue( true ); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /demo/js/crdt_gset.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | // A CRDT grow-only set 4 | var node = require('./node'); 5 | 6 | // Our set of elements 7 | var crdt = new Set(); 8 | 9 | // Serialize a CRDT to something that'll transport as JSON 10 | function serialize(crdt) { 11 | return Array.from(crdt); 12 | } 13 | 14 | // Deserialize a JSON-transported structure to a CRDT 15 | function deserialize(obj) { 16 | return new Set(obj); 17 | } 18 | 19 | // Merge two CRDTs 20 | function merge(a, b) { 21 | const merged = new Set(); 22 | for (const x of a) { 23 | merged.add(x); 24 | } 25 | for (const x of b) { 26 | merged.add(x); 27 | } 28 | return merged; 29 | }; 30 | 31 | // Add new elements to our local state 32 | node.on('add', function(req) { 33 | crdt.add(req.body.element); 34 | console.warn ("state after add:", crdt); 35 | node.reply(req, {type: 'add_ok'}); 36 | }); 37 | 38 | // When we get a read request, return our messages 39 | node.on('read', function(req) { 40 | node.reply(req, {type: 'read_ok', value: Array.from(crdt)}); 41 | }); 42 | 43 | // When we receive a replication message, merge it into our CRDT 44 | node.on('replicate', (req) => { 45 | crdt = merge(crdt, deserialize(req.body.value)); 46 | console.warn("state after replicate:", crdt); 47 | }); 48 | 49 | // When we initialize, start a replication loop 50 | node.on('init', (req) => { 51 | setInterval(() => { 52 | console.warn('Replicate!'); 53 | for (const peer of node.nodeIds()) { 54 | if (peer !== node.nodeId()) { 55 | node.send(peer, {type: 'replicate', value: serialize(crdt)}); 56 | } 57 | } 58 | }, 5000); 59 | }); 60 | 61 | node.main(); 62 | -------------------------------------------------------------------------------- /demo/js/crdt_pn_counter.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | // A CRDT PN-counter 4 | var node = require('./node'); 5 | 6 | class GCounter { 7 | // Takes a map of node names to the count on that node. 8 | constructor(counts) { 9 | this.counts = counts; 10 | } 11 | 12 | // Returns the effective value of the counter. 13 | value() { 14 | var total = 0; 15 | for (const node in this.counts) { 16 | total += this.counts[node]; 17 | } 18 | return total; 19 | } 20 | 21 | // Merges another GCounter into this one 22 | merge(other) { 23 | const counts = {... this.counts}; 24 | for (const node in other.counts) { 25 | if (counts[node] == undefined) { 26 | counts[node] = other.counts[node]; 27 | } else { 28 | counts[node] = Math.max(this.counts[node], other.counts[node]); 29 | } 30 | } 31 | return new GCounter(counts); 32 | } 33 | 34 | // Convert to a JSON-serializable object 35 | toJSON() { 36 | return this.counts; 37 | } 38 | 39 | // Inflates a JSON-serialized object back into a fresh GCounter 40 | fromJSON(json) { 41 | return new GCounter(json); 42 | } 43 | 44 | // Increment by delta 45 | increment(node, delta) { 46 | var count = this.counts[node]; 47 | if (count == undefined) { 48 | count = 0; 49 | } 50 | var counts = {... this.counts}; 51 | counts[node] = count + delta; 52 | return new GCounter(counts); 53 | } 54 | } 55 | 56 | class PNCounter { 57 | // Takes an increment GCounter and a decrement GCounter 58 | constructor(plus, minus) { 59 | this.plus = plus; 60 | this.minus = minus; 61 | } 62 | 63 | // The effective value is all increments minus decrements 64 | value() { 65 | return this.plus.value() - this.minus.value(); 66 | } 67 | 68 | // Merges another PNCounter into this one 69 | merge(other) { 70 | return new PNCounter( 71 | this.plus.merge(other.plus), 72 | this.minus.merge(other.minus) 73 | ); 74 | } 75 | 76 | // Converts to a JSON-serializable object 77 | toJSON() { 78 | return {plus: this.plus, minus: this.minus}; 79 | } 80 | 81 | // Inflates a JSON-serialized object back into a fresh PNCounter 82 | fromJSON(json) { 83 | return new PNCounter( 84 | this.plus.fromJSON(json.plus), 85 | this.minus.fromJSON(json.minus) 86 | ); 87 | } 88 | 89 | // Increment by delta 90 | increment(node, delta) { 91 | if (0 < delta) { 92 | return new PNCounter(this.plus.increment(node, delta), this.minus); 93 | } else { 94 | return new PNCounter(this.plus, this.minus.increment(node, delta * -1)); 95 | } 96 | } 97 | } 98 | 99 | // Our CRDT state 100 | var crdt = new PNCounter(new GCounter({}), new GCounter({})); 101 | 102 | // Add new elements to our local state 103 | node.on('add', function(req) { 104 | crdt = crdt.increment(node.nodeId(), req.body.delta); 105 | console.warn ("state after add:", crdt); 106 | node.reply(req, {type: 'add_ok'}); 107 | }); 108 | 109 | // When we get a read request, return our messages 110 | node.on('read', function(req) { 111 | node.reply(req, {type: 'read_ok', value: crdt.value()}); 112 | }); 113 | 114 | // When we receive a replication message, merge it into our CRDT 115 | node.on('replicate', (req) => { 116 | crdt = crdt.merge(crdt.fromJSON(req.body.value)); 117 | console.warn("state after replicate:", crdt); 118 | }); 119 | 120 | // When we initialize, start a replication loop 121 | node.on('init', (req) => { 122 | setInterval(() => { 123 | console.warn('Replicate!'); 124 | for (const peer of node.nodeIds()) { 125 | if (peer !== node.nodeId()) { 126 | node.send(peer, {type: 'replicate', value: crdt.toJSON()}); 127 | } 128 | } 129 | }, 5000); 130 | }); 131 | 132 | node.main(); 133 | -------------------------------------------------------------------------------- /demo/js/echo.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | // A basic echo server 4 | var node = require('./node'); 5 | 6 | node.on('echo', function(req) { 7 | node.reply(req, {type: 'echo_ok', 'echo': req.body.echo}); 8 | }); 9 | 10 | node.main(); 11 | -------------------------------------------------------------------------------- /demo/js/echo_minimal.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | var readline = require('readline'); 4 | var rl = readline.createInterface({ 5 | input: process.stdin, 6 | output: process.stdout, 7 | terminal: false 8 | }); 9 | 10 | // Local state 11 | var nodeId; 12 | var nodeIds; 13 | var nextMsgId = 0; 14 | 15 | // Mainloop 16 | rl.on('line', function(line) { 17 | console.warn("got", line); 18 | handle(JSON.parse(line)); 19 | }); 20 | 21 | // Send a message body to the given destination STDOUT 22 | function sendMsg(dest, body) { 23 | let msg = {src: nodeId, dest: dest, body: body}; 24 | console.log(JSON.stringify(msg)); 25 | } 26 | 27 | // Reply to the given request message with the given response body. 28 | function reply(req, resBody) { 29 | let body = {... resBody, in_reply_to: req.body.msg_id}; 30 | sendMsg(req.src, body); 31 | } 32 | 33 | // Handle a request message from stdin 34 | function handle(req) { 35 | switch (req.body.type) { 36 | case 'init': handleInit(req); break; 37 | case 'echo': handleEcho(req); break; 38 | default: 39 | console.warn("Don't know how to handle message of type", req.body.type, 40 | req); 41 | } 42 | } 43 | 44 | // Handle an initialization message 45 | function handleInit(req) { 46 | let body = req.body; 47 | nodeId = body.node_id; 48 | nodeIds = body.node_ids; 49 | console.warn('I am node', nodeId); 50 | reply(req, {type: 'init_ok'}); 51 | } 52 | 53 | // Handle an echo message 54 | function handleEcho(req) { 55 | reply(req, {type: 'echo_ok', echo: req.body.echo}); 56 | } 57 | -------------------------------------------------------------------------------- /demo/js/gossip.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | // A gossip system which supports the Maelstrom broadcast workload 4 | var node = require('./node'); 5 | 6 | // Our local peers: an array of nodes. 7 | var peers; 8 | 9 | // Our set of messages received. 10 | var messages = new Set(); 11 | 12 | // Save our topology when it arrives 13 | node.on('topology', function(req) { 14 | peers = req.body.topology[node.nodeId()]; 15 | console.warn("My peers are", peers); 16 | node.reply(req, {type: 'topology_ok'}); 17 | }); 18 | 19 | // When we get a read request, return our messages 20 | node.on('read', function(req) { 21 | node.reply(req, {type: 'read_ok', messages: Array.from(messages)}); 22 | }); 23 | 24 | // When we get a broadcast, add it to the message set and broadcast it to peers 25 | node.on('broadcast', function(req) { 26 | let m = req.body.message; 27 | if (! messages.has(m)) { 28 | // We haven't seen this yet; save it 29 | messages.add(m); 30 | // Broadcast to peers except the one who sent it to us 31 | for (const peer of peers) { 32 | if (peer !== req.src) { 33 | node.retryRPC(peer, {type: 'broadcast', message: m}); 34 | } 35 | } 36 | } 37 | node.reply(req, {type: 'broadcast_ok'}); 38 | }); 39 | 40 | node.main(); 41 | -------------------------------------------------------------------------------- /demo/js/single_key_txn.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | // A simple list-append transaction service which stores data in a single key 4 | // in lin-kv. 5 | var node = require('./node'); 6 | 7 | // The service we store the state in 8 | const storage = 'lin-kv'; 9 | // The key we store state in 10 | const storageKey = 'state'; 11 | 12 | // Our in-memory state: a map of keys to values. 13 | var state = new Map(); 14 | 15 | // Serialize a map to JSON 16 | function serializeMap(m) { 17 | const pairs = []; 18 | m.forEach((v, k) => { 19 | pairs.push(k); 20 | pairs.push(v); 21 | }); 22 | return pairs; 23 | } 24 | 25 | // Deserialize a map from JSON 26 | function deserializeMap(pairs) { 27 | const m = new Map(); 28 | for (var i = 0; i < pairs.length; i += 2) { 29 | m.set(pairs[i], pairs[i + 1]); 30 | } 31 | return m; 32 | } 33 | 34 | // Copies a map 35 | function copyMap(m) { 36 | const m2 = new Map(); 37 | m.forEach((v, k) => { 38 | m2.set(k, v); 39 | }); 40 | return m2; 41 | } 42 | 43 | // Fetches k from storage, returning a default state if it does not exist. 44 | async function getKey(k, notFound) { 45 | try { 46 | const body = await node.rpc(storage, {type: 'read', key: k}); 47 | return body.value; 48 | } catch (err) { 49 | if (err.code === 20) { 50 | return notFound; 51 | } else { 52 | throw err; 53 | } 54 | } 55 | } 56 | 57 | // Fetch the DB state from the storage service 58 | async function getState() { 59 | const pairs = await getKey(storageKey, []); 60 | return deserializeMap(pairs); 61 | } 62 | 63 | // Atomically change the state from state to state2. 64 | async function casState(state, state2) { 65 | try { 66 | await node.rpc(storage, { 67 | type: 'cas', 68 | key: storageKey, 69 | from: serializeMap(state), 70 | to: serializeMap(state2), 71 | create_if_not_exists: true 72 | }); 73 | } catch (err) { 74 | if (err.code === 22) { 75 | throw {code: 30, text: "state changed during txn"}; 76 | } else { 77 | throw err; 78 | } 79 | } 80 | } 81 | 82 | // Apply a transaction to a state, returning a [state', txn'] pair. 83 | function applyTxn(state, txn) { 84 | const state2 = copyMap(state); 85 | const txn2 = []; 86 | for (mop of txn) { 87 | const [f, k, v] = mop; 88 | const value = state2.get(k); 89 | switch (f) { 90 | case "r": 91 | txn2.push([f, k, value]); 92 | break; 93 | case "append": 94 | let value2; 95 | if (value === undefined) { 96 | value2 = [v]; 97 | } else { 98 | value2 = [... value, v]; 99 | } 100 | state2.set(k, value2); 101 | txn2.push(mop); 102 | break; 103 | } 104 | } 105 | return [state2, txn2]; 106 | } 107 | 108 | node.on('txn', async function(req) { 109 | const state = await getState(); 110 | const [state2, txn2] = await applyTxn(state, req.body.txn); 111 | await casState(state, state2); 112 | node.reply(req, {type: 'txn_ok', 'txn': txn2}); 113 | }); 114 | 115 | node.main(); 116 | -------------------------------------------------------------------------------- /demo/python/README.md: -------------------------------------------------------------------------------- 1 | # maelstrom-python 2 | 3 | This is a Python implementation of the Maelstrom Node. This provides basic 4 | message handling and an asyncio event loop. 5 | 6 | ## Usage 7 | 8 | For Python, the easiest way to run a script is to make sure that it has the 9 | executable mode set (`chmod +x`) and add a shebang line to the top of the file. 10 | 11 | ```python 12 | #!/usr/bin/env python3 13 | ``` 14 | 15 | Examples are in `echo.py` and `broadcast.py`, which import from `maelstrom.py`. 16 | 17 | ```sh 18 | $ maelstrom test -w echo --bin echo.py ... 19 | $ maelstrom test -w broadcast --bin broadcast.py ... 20 | ``` 21 | -------------------------------------------------------------------------------- /demo/python/broadcast.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import asyncio 4 | from maelstrom import Node, Body, Request 5 | 6 | node = Node() 7 | 8 | messages: set[int] = set() 9 | cond = asyncio.Condition() 10 | 11 | 12 | async def add_messages(values: set[int]) -> None: 13 | if values <= messages: 14 | return 15 | messages.update(values) 16 | async with cond: 17 | cond.notify_all() 18 | 19 | 20 | @node.handler 21 | async def broadcast(req: Request) -> Body: 22 | msg = req.body["message"] 23 | await add_messages({msg}) 24 | return {"type": "broadcast_ok"} 25 | 26 | 27 | @node.handler 28 | async def broadcast_many(req: Request) -> Body: 29 | msgs = req.body["messages"] 30 | await add_messages(set(msgs)) 31 | return {"type": "broadcast_many_ok"} 32 | 33 | 34 | @node.handler 35 | async def read(req: Request) -> Body: 36 | return {"type": "read_ok", "messages": list(messages)} 37 | 38 | 39 | @node.handler 40 | async def topology(req: Request) -> Body: 41 | neighbors = req.body["topology"][node.node_id] 42 | for n in neighbors: 43 | node.spawn(gossip_task(n)) 44 | return {"type": "topology_ok"} 45 | 46 | 47 | async def gossip_task(neighbor: str) -> None: 48 | sent = set() 49 | while True: 50 | # assert sent <= messages 51 | if len(sent) == len(messages): 52 | async with cond: 53 | # Wait for the next update to our message set. 54 | await cond.wait_for(lambda: len(sent) != len(messages)) 55 | 56 | to_send = messages - sent 57 | body = {"type": "broadcast_many", "messages": list(to_send)} 58 | resp = await node.rpc(neighbor, body) 59 | if resp["type"] == "broadcast_many_ok": 60 | sent.update(to_send) 61 | 62 | 63 | node.run() 64 | -------------------------------------------------------------------------------- /demo/python/echo.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from maelstrom import Node, Body, Request 4 | 5 | node = Node() 6 | 7 | 8 | @node.handler 9 | async def echo(req: Request) -> Body: 10 | return {"type": "echo_ok", "echo": req.body["echo"]} 11 | 12 | 13 | node.run() 14 | -------------------------------------------------------------------------------- /demo/ruby/broadcast.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/ruby 2 | 3 | # A basic broadcast system. Keeps track of a set of messages, inserted via 4 | # `broadcast` requests from clients. 5 | 6 | require_relative 'node.rb' 7 | require 'set' 8 | 9 | class BroadcastNode 10 | def initialize 11 | @node = Node.new 12 | @neighbors = [] 13 | @lock = Mutex.new 14 | @messages = Set.new 15 | 16 | @node.on "topology" do |msg| 17 | @neighbors = msg[:body][:topology][@node.node_id.to_sym] 18 | @node.log "My neighbors are #{@neighbors.inspect}" 19 | @node.reply! msg, {type: "topology_ok"} 20 | end 21 | 22 | @node.on "read" do |msg| 23 | @lock.synchronize do 24 | @node.reply! msg, {type: "read_ok", 25 | messages: @messages.to_a} 26 | end 27 | end 28 | 29 | @node.on "broadcast" do |msg| 30 | m = msg[:body][:message] 31 | @lock.synchronize do 32 | unless @messages.include? m 33 | @messages.add m 34 | @node.log "messages now #{@messages}" 35 | 36 | # Broadcast this message to neighbors (except whoever sent it to us) 37 | @node.other_node_ids.each do |neighbor| 38 | unless neighbor == msg[:src] 39 | @node.rpc! neighbor, {type: "broadcast", message: m} do |res| 40 | # Eh, whatever 41 | end 42 | end 43 | end 44 | end 45 | end 46 | @node.reply! msg, {type: "broadcast_ok"} 47 | end 48 | end 49 | 50 | def main! 51 | @node.main! 52 | end 53 | end 54 | 55 | BroadcastNode.new.main! 56 | -------------------------------------------------------------------------------- /demo/ruby/crdt.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/ruby 2 | 3 | # Sets up a Node to act as a generic CRDT server. Takes a CRDT value with four 4 | # methods: 5 | # 6 | # .from_json(json) Inflates a value (ignoring dt) from a JSON structure 7 | # .to_json Returns a JSON structure representation for serialization 8 | # .merge(other) Returns a copy of value merged with other. 9 | # .read Returns the effective state of this CRDT value. 10 | # 11 | # As a node, supports the following messages: 12 | # 13 | # {type: "read"} returns the current value of the valuetype 14 | # {type: "merge", :value value} merges a value into our own 15 | # 16 | # The current value of the CRDT is available in crdt.value. 17 | 18 | class CRDT 19 | attr_accessor :node, :value 20 | 21 | def initialize(node, value) 22 | @node = node 23 | @value = value 24 | 25 | @node.on "read" do |msg| 26 | @node.reply! msg, type: "read_ok", value: @value.read 27 | end 28 | 29 | # Merge another node's value into our own 30 | @node.on "merge" do |msg| 31 | @value = @value.merge @value.from_json(msg[:body][:value]) 32 | STDERR.puts "Value now #{@value.to_json}" 33 | @node.reply! msg, type: "merge_ok" 34 | end 35 | 36 | # Silently ignore merge_ok messages 37 | @node.on "merge_ok" do |msg| 38 | end 39 | 40 | # Periodically replicate entire state 41 | @node.every 5 do 42 | @node.other_node_ids.each do |node| 43 | @node.send! node, {type: "merge", value: @value.to_json} 44 | end 45 | end 46 | end 47 | end 48 | -------------------------------------------------------------------------------- /demo/ruby/echo.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'json' 4 | 5 | class EchoServer 6 | def initialize 7 | @node_id = nil 8 | @next_msg_id = 0 9 | end 10 | 11 | def reply!(request, body) 12 | id = @next_msg_id += 1 13 | body = body.merge msg_id: id, in_reply_to: request[:body][:msg_id] 14 | msg = {src: @node_id, dest: request[:src], body: body} 15 | JSON.dump msg, STDOUT 16 | STDOUT << "\n" 17 | STDOUT.flush 18 | end 19 | 20 | def main! 21 | STDERR.puts "Online" 22 | 23 | while line = STDIN.gets 24 | req = JSON.parse line, symbolize_names: true 25 | STDERR.puts "Received #{req.inspect}" 26 | 27 | body = req[:body] 28 | case body[:type] 29 | # Initialize this node 30 | when "init" 31 | @node_id = body[:node_id] 32 | STDERR.puts "Initialized node #{@node_id}" 33 | reply! req, {type: :init_ok} 34 | 35 | # Send echoes back 36 | when "echo" 37 | STDERR.puts "Echoing #{body}" 38 | reply! req, {type: "echo_ok", echo: body[:echo]} 39 | end 40 | end 41 | end 42 | end 43 | 44 | EchoServer.new.main! 45 | -------------------------------------------------------------------------------- /demo/ruby/echo_full.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/ruby 2 | 3 | # A simple echo server 4 | 5 | require_relative 'node.rb' 6 | 7 | class EchoNode 8 | def initialize 9 | @node = Node.new 10 | 11 | @node.on "echo" do |msg| 12 | @node.reply! msg, msg[:body].merge(type: "echo_ok") 13 | end 14 | end 15 | 16 | def main! 17 | @node.main! 18 | end 19 | end 20 | 21 | EchoNode.new.main! 22 | -------------------------------------------------------------------------------- /demo/ruby/errors.rb: -------------------------------------------------------------------------------- 1 | class RPCError < StandardError 2 | class << self 3 | def timeout(msg); new 0, msg; end 4 | 5 | def not_supported(msg); new 10, msg; end 6 | def temporarily_unavailable(msg); new 11, msg; end 7 | def malformed_request(msg); new 12, msg; end 8 | def crash(msg); new 13, msg; end 9 | def abort(msg); new 14, msg; end 10 | 11 | def key_does_not_exist(msg); new 20, msg; end 12 | def key_already_exists(msg); new 21, msg; end 13 | def precondition_failed(msg); new 22, msg; end 14 | 15 | def txn_conflict(msg); new 30, msg; end 16 | end 17 | 18 | attr_reader :code 19 | 20 | def initialize(code, text) 21 | @code = code 22 | @text = text 23 | super(text) 24 | end 25 | 26 | # Constructs a JSON error response 27 | def to_json 28 | {type: "error", 29 | code: @code, 30 | text: @text} 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /demo/ruby/g_set.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/ruby 2 | 3 | # A CRDT-based grow-only set. 4 | 5 | require_relative 'node.rb' 6 | require 'set' 7 | 8 | class GSetNode 9 | def initialize 10 | @node = Node.new 11 | @set = Set.new 12 | 13 | @node.on "read" do |msg| 14 | @node.reply! msg, {type: "read_ok", value: @set.to_a} 15 | end 16 | 17 | @node.on "add" do |msg| 18 | element = msg[:body][:element] 19 | @set.add element 20 | @node.reply! msg, {type: "add_ok"} 21 | end 22 | 23 | # Accept a single element from another node 24 | @node.on "replicate_one" do |msg| 25 | @set.add msg[:body][:element] 26 | end 27 | 28 | # Accept an entire value from another node 29 | @node.on "replicate_full" do |msg| 30 | @set |= msg[:body][:value] 31 | end 32 | 33 | # Periodically replicate entire state 34 | @node.every 5 do 35 | STDERR.puts "Replicating!" 36 | @node.other_node_ids.each do |node| 37 | @node.send! node, ({type: "replicate_full", value: @set.to_a}) 38 | end 39 | end 40 | end 41 | 42 | def main! 43 | @node.main! 44 | end 45 | end 46 | 47 | GSetNode.new.main! 48 | -------------------------------------------------------------------------------- /demo/ruby/lin_kv_proxy.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/ruby 2 | 3 | # A linearizable key-value store which works by proxying all requests to a 4 | # Maelstrom-provided linearizable kv store. 5 | 6 | require_relative 'node.rb' 7 | 8 | class LinKVNode 9 | def initialize 10 | @node = Node.new 11 | 12 | @node.on "read" do |msg| 13 | proxy! msg 14 | end 15 | 16 | @node.on "write" do |msg| 17 | proxy! msg 18 | end 19 | 20 | @node.on "cas" do |msg| 21 | proxy! msg 22 | end 23 | end 24 | 25 | # Takes a client request, proxies the body to lin_kv, and sends the response 26 | # back to the client. 27 | def proxy!(req_msg) 28 | proxy_body = req_msg[:body].clone 29 | proxy_body.delete :msg_id 30 | 31 | # Replace with seq-kv or lww-kv to see linearization failures! 32 | # @node.rpc! "seq-kv", proxy_body do |res| 33 | @node.rpc! "lin-kv", proxy_body do |res| 34 | res[:body].delete :msg_id 35 | @node.reply! req_msg, res[:body] 36 | end 37 | end 38 | 39 | def main! 40 | @node.main! 41 | end 42 | end 43 | 44 | LinKVNode.new.main! 45 | -------------------------------------------------------------------------------- /demo/ruby/node.rb: -------------------------------------------------------------------------------- 1 | require 'json' 2 | require_relative 'errors.rb' 3 | require_relative 'promise.rb' 4 | 5 | class Node 6 | attr_reader :node_id, :node_ids 7 | 8 | def initialize 9 | @node_id = nil 10 | @node_ids = nil 11 | @next_msg_id = 0 12 | 13 | @init_handlers = [] 14 | @handlers = {} 15 | @callbacks = {} 16 | @periodic_tasks = [] 17 | 18 | @lock = Monitor.new 19 | @log_lock = Mutex.new 20 | 21 | # Register an initial handler for the init message 22 | on "init" do |msg| 23 | # Set our node ID and IDs 24 | @node_id = msg[:body][:node_id] 25 | @node_ids = msg[:body][:node_ids] 26 | 27 | @init_handlers.each do |h| 28 | h.call msg 29 | end 30 | 31 | reply! msg, type: "init_ok" 32 | log "Node #{@node_id} initialized" 33 | 34 | # Launch threads for periodic tasks 35 | start_periodic_tasks! 36 | end 37 | end 38 | 39 | # Writes a message to stderr 40 | def log(message) 41 | @log_lock.synchronize do 42 | STDERR.puts message 43 | STDERR.flush 44 | end 45 | end 46 | 47 | # Register a new message type handler 48 | def on(type, &handler) 49 | if @handlers[type] 50 | raise "Already have a handler for #{type}!" 51 | end 52 | 53 | @handlers[type] = handler 54 | end 55 | 56 | # A special handler for initialization 57 | def on_init(&handler) 58 | @init_handlers << handler 59 | end 60 | 61 | # Periodically evaluates block every dt seconds with the node lock 62 | # held--helpful for building periodic replication tasks, timeouts, etc. 63 | def every(dt, &block) 64 | @periodic_tasks << {dt: dt, f: block} 65 | end 66 | 67 | # Send a body to the given node id. Fills in src with our own node_id. 68 | def send!(dest, body) 69 | msg = {dest: dest, 70 | src: @node_id, 71 | body: body} 72 | @lock.synchronize do 73 | log "Sent #{msg.inspect}" 74 | JSON.dump msg, STDOUT 75 | STDOUT << "\n" 76 | STDOUT.flush 77 | end 78 | end 79 | 80 | # Broadcasts a message to all other nodes. 81 | def broadcast!(body) 82 | other_node_ids.each do |n| 83 | send! n, body 84 | end 85 | end 86 | 87 | # Reply to a request with a response body 88 | def reply!(req, body) 89 | body = body.merge({in_reply_to: req[:body][:msg_id]}) 90 | send! req[:src], body 91 | end 92 | 93 | # Send an async RPC request. Invokes block with response message once one 94 | # arrives. 95 | def rpc!(dest, body, &handler) 96 | @lock.synchronize do 97 | msg_id = @next_msg_id += 1 98 | @callbacks[msg_id] = handler 99 | body = body.merge({msg_id: msg_id}) 100 | send! dest, body 101 | end 102 | end 103 | 104 | def other_node_ids 105 | @node_ids.reject do |id| 106 | id == @node_id 107 | end 108 | end 109 | 110 | # Sends a broadcast RPC request. Invokes block with a response message for 111 | # each response that arrives. 112 | def brpc!(body, &handler) 113 | other_node_ids.each do |node| 114 | rpc! node, body, &handler 115 | end 116 | end 117 | 118 | # Sends a synchronous RPC request, blocking this thread and returning the 119 | # response message. 120 | def sync_rpc!(dest, body) 121 | p = Promise.new 122 | rpc! dest, body do |response| 123 | p.deliver! response 124 | end 125 | p.await 126 | end 127 | 128 | # Launches threads to process periodic handlers 129 | def start_periodic_tasks! 130 | @periodic_tasks.each do |task| 131 | Thread.new do 132 | loop do 133 | task[:f].call 134 | sleep task[:dt] 135 | end 136 | end 137 | end 138 | end 139 | 140 | # Turns a line of STDIN into a message hash 141 | def parse_msg(line) 142 | msg = JSON.parse line 143 | msg.transform_keys!(&:to_sym) 144 | msg[:body].transform_keys!(&:to_sym) 145 | msg 146 | end 147 | 148 | # Loops, processing messages from STDIN 149 | def main! 150 | Thread.abort_on_exception = true 151 | 152 | while line = STDIN.gets 153 | msg = parse_msg line 154 | log "Received #{msg.inspect}" 155 | 156 | # What handler should we use for this message? 157 | handler = nil 158 | @lock.synchronize do 159 | if in_reply_to = msg[:body][:in_reply_to] 160 | if handler = @callbacks[msg[:body][:in_reply_to]] 161 | @callbacks.delete msg[:body][:in_reply_to] 162 | else 163 | log "Ignoring reply to #{in_reply_to} with no callback" 164 | end 165 | elsif handler = @handlers[msg[:body][:type]] 166 | else 167 | raise "No handler for #{msg.inspect}" 168 | end 169 | end 170 | 171 | if handler 172 | # Actually handle message 173 | Thread.new(handler, msg) do |handler, msg| 174 | begin 175 | handler.call msg 176 | rescue RPCError => e 177 | reply! msg, e.to_json 178 | rescue => e 179 | log "Exception handling #{msg}:\n#{e.full_message}" 180 | reply! msg, RPCError.crash(e.full_message).to_json 181 | end 182 | end 183 | end 184 | end 185 | end 186 | end 187 | -------------------------------------------------------------------------------- /demo/ruby/pn_counter.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/ruby 2 | 3 | # A single PN counter. 4 | 5 | require_relative 'node.rb' 6 | require_relative 'crdt.rb' 7 | 8 | class GCounter 9 | attr_reader :counters 10 | 11 | # We represent a G-counter as a map of node_id => value_at_node 12 | def initialize(counters = {}) 13 | @counters = counters 14 | end 15 | 16 | def from_json(json) 17 | GCounter.new json 18 | end 19 | 20 | def to_json 21 | @counters 22 | end 23 | 24 | # The value of a G-counter is the sum of its values. 25 | def read 26 | @counters.values.reduce(0) do |sum, x| 27 | sum + x 28 | end 29 | end 30 | 31 | # Adding a value to a counter means incrementing the value for this 32 | # node_id. 33 | def add(node_id, delta) 34 | counters = @counters.dup 35 | counters[node_id] = (counters[node_id] || 0) + delta 36 | GCounter.new counters 37 | end 38 | 39 | # Merging two G-counters means taking the maxima of corresponding hash 40 | # elements. 41 | def merge(other) 42 | GCounter.new(@counters.merge(other.counters) { |k, v1, v2| 43 | [v1, v2].max 44 | }) 45 | end 46 | end 47 | 48 | class PNCounter 49 | # A PN-Counter combines a G-counter for increments and another for 50 | # decrements 51 | attr_reader :inc, :dec 52 | 53 | def initialize(inc = GCounter.new, dec = GCounter.new) 54 | @inc = inc 55 | @dec = dec 56 | end 57 | 58 | def from_json(json) 59 | PNCounter.new(@inc.from_json(json["inc"]), 60 | @dec.from_json(json["dec"])) 61 | end 62 | 63 | def to_json 64 | {inc: @inc.to_json, 65 | dec: @dec.to_json} 66 | end 67 | 68 | # The value of the G-counter is the increments minus the decrements 69 | def read 70 | @inc.read - @dec.read 71 | end 72 | 73 | # Adding a delta to the counter means modifying the value in one of the 74 | # g-counters. 75 | def add(node_id, delta) 76 | if 0 <= delta 77 | PNCounter.new @inc.add(node_id, delta), @dec 78 | else 79 | PNCounter.new @inc, @dec.add(node_id, -delta) 80 | end 81 | end 82 | 83 | # Merging PN-Counters just means merging the incs and decs 84 | def merge(other) 85 | PNCounter.new @inc.merge(other.inc), @dec.merge(other.dec) 86 | end 87 | end 88 | 89 | class PNCounterNode 90 | def initialize 91 | @node = Node.new 92 | @counter = CRDT.new(@node, PNCounter.new) 93 | 94 | # We can take an add message with a `delta` integer, and 95 | # increment/decrement our entry in the counter. 96 | @node.on "add" do |msg| 97 | @counter.value = @counter.value.add @node.node_id, msg[:body][:delta] 98 | STDERR.puts "Value now #{@counter.value.to_json}" 99 | @node.reply! msg, {type: "add_ok"} 100 | end 101 | end 102 | 103 | def main! 104 | @node.main! 105 | end 106 | end 107 | 108 | PNCounterNode.new.main! 109 | -------------------------------------------------------------------------------- /demo/ruby/promise.rb: -------------------------------------------------------------------------------- 1 | require_relative "errors.rb" 2 | 3 | class Promise 4 | WAITING = Object.new 5 | TIMEOUT = 5 6 | 7 | def initialize 8 | @lock = Mutex.new 9 | @cvar = ConditionVariable.new 10 | @value = WAITING 11 | end 12 | 13 | # Blocks this thread until a value is delivered, then returns it. 14 | def await 15 | if @value != WAITING 16 | return @value 17 | end 18 | 19 | # Not ready yet; block 20 | @lock.synchronize do 21 | @cvar.wait @lock, TIMEOUT 22 | end 23 | 24 | if @value != WAITING 25 | return @value 26 | else 27 | raise RPCError.timeout "promise timed out" 28 | end 29 | end 30 | 31 | # Delivers value. We don't check for double-delivery--this is a super 32 | # bare-bones implementation and we're trying to minimize code! 33 | def deliver!(value) 34 | @value = value 35 | # Not sure if we actually have to hold the mutex here. The cvar docs are... 36 | # vague. 37 | @lock.synchronize do 38 | @cvar.broadcast 39 | end 40 | self 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /demo/rust/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | authors = ["Ivan Prisyazhnyy "] 3 | categories = ["asynchronous", "concurrency", "network-programming", "distributed-systems", "simulation"] 4 | description = "Maelstrom Rust node framework" 5 | edition = "2021" 6 | keywords = ["maelstrom", "fly-io", "distributed-systems", "testing"] 7 | readme = "README.md" 8 | repository = "https://github.com/jepsen-io/maelstrom" 9 | version = "0.1.0" 10 | name = "maelstrom-rust-demo" 11 | 12 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 13 | 14 | [dependencies] 15 | async-trait = "0.1.68" 16 | env_logger = "0.10.0" 17 | log = "0.4.17" 18 | serde = { version = "1.0.159", features = ["derive"] } 19 | serde_json = "1.0.95" 20 | tokio = { version = "1.27.0", features = ["full"] } 21 | tokio-context = "0.1.3" 22 | maelstrom-node="0.1.6" 23 | 24 | [[bin]] 25 | name = "echo" 26 | 27 | [[bin]] 28 | name = "broadcast" 29 | 30 | [[bin]] 31 | name = "lin_kv" 32 | 33 | [[bin]] 34 | name = "g_set" 35 | -------------------------------------------------------------------------------- /demo/rust/src/bin/broadcast.rs: -------------------------------------------------------------------------------- 1 | /// ```bash 2 | /// $ cargo build 3 | /// $ RUST_LOG=debug maelstrom test -w broadcast --bin ./target/debug/broadcast --node-count 2 --time-limit 20 --rate 10 --log-stderr 4 | /// ```` 5 | use async_trait::async_trait; 6 | use log::info; 7 | use maelstrom::protocol::Message; 8 | use maelstrom::{done, Node, Result, Runtime}; 9 | use serde::{Deserialize, Serialize}; 10 | use std::collections::{HashMap, HashSet}; 11 | use std::sync::{Arc, Mutex}; 12 | 13 | pub(crate) fn main() -> Result<()> { 14 | Runtime::init(try_main()) 15 | } 16 | 17 | async fn try_main() -> Result<()> { 18 | let handler = Arc::new(Handler::default()); 19 | Runtime::new().with_handler(handler).run().await 20 | } 21 | 22 | #[derive(Clone, Default)] 23 | struct Handler { 24 | inner: Arc>, 25 | } 26 | 27 | #[derive(Clone, Default)] 28 | struct Inner { 29 | s: HashSet, 30 | t: Vec, 31 | } 32 | 33 | #[async_trait] 34 | impl Node for Handler { 35 | async fn process(&self, runtime: Runtime, req: Message) -> Result<()> { 36 | let msg: Result = req.body.as_obj(); 37 | match msg { 38 | Ok(Request::Read {}) => { 39 | let data = self.snapshot(); 40 | let msg = Response::ReadOk { messages: data }; 41 | return runtime.reply(req, msg).await; 42 | } 43 | Ok(Request::Broadcast { message: element }) => { 44 | if self.try_add(element) { 45 | info!("messages now {}", element); 46 | for node in runtime.neighbours() { 47 | runtime.call_async(node, Request::Broadcast { message: element }); 48 | } 49 | } 50 | 51 | return runtime.reply_ok(req).await; 52 | } 53 | Ok(Request::Topology { topology }) => { 54 | let neighbours = topology.get(runtime.node_id()).unwrap(); 55 | self.inner.lock().unwrap().t = neighbours.clone(); 56 | info!("My neighbors are {:?}", neighbours); 57 | return runtime.reply_ok(req).await; 58 | } 59 | _ => done(runtime, req), 60 | } 61 | } 62 | } 63 | 64 | impl Handler { 65 | fn snapshot(&self) -> Vec { 66 | self.inner.lock().unwrap().s.iter().copied().collect() 67 | } 68 | 69 | fn try_add(&self, val: u64) -> bool { 70 | let mut g = self.inner.lock().unwrap(); 71 | if !g.s.contains(&val) { 72 | g.s.insert(val); 73 | return true; 74 | } 75 | false 76 | } 77 | } 78 | 79 | #[derive(Serialize, Deserialize)] 80 | #[serde(rename_all = "snake_case", tag = "type")] 81 | enum Request { 82 | Init {}, 83 | Read {}, 84 | Broadcast { 85 | message: u64, 86 | }, 87 | Topology { 88 | topology: HashMap>, 89 | }, 90 | } 91 | 92 | #[derive(Serialize, Deserialize)] 93 | #[serde(rename_all = "snake_case", tag = "type")] 94 | enum Response { 95 | ReadOk { 96 | messages: Vec, 97 | }, 98 | } 99 | -------------------------------------------------------------------------------- /demo/rust/src/bin/echo.rs: -------------------------------------------------------------------------------- 1 | /// ```bash 2 | /// $ cargo build 3 | /// $ maelstrom test -w echo --bin ./target/debug/echo --node-count 1 --time-limit 10 --log-stderr 4 | /// ```` 5 | use async_trait::async_trait; 6 | use maelstrom::protocol::Message; 7 | use maelstrom::{done, Node, Result, Runtime}; 8 | use std::sync::Arc; 9 | 10 | pub(crate) fn main() -> Result<()> { 11 | Runtime::init(try_main()) 12 | } 13 | 14 | async fn try_main() -> Result<()> { 15 | let handler = Arc::new(Handler::default()); 16 | Runtime::new().with_handler(handler).run().await 17 | } 18 | 19 | #[derive(Clone, Default)] 20 | struct Handler {} 21 | 22 | #[async_trait] 23 | impl Node for Handler { 24 | async fn process(&self, runtime: Runtime, req: Message) -> Result<()> { 25 | if req.get_type() == "echo" { 26 | let echo = req.body.clone().with_type("echo_ok"); 27 | return runtime.reply(req, echo).await; 28 | } 29 | 30 | done(runtime, req) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /demo/rust/src/bin/g_set.rs: -------------------------------------------------------------------------------- 1 | /// ```bash 2 | /// $ cargo build 3 | /// $ RUST_LOG=debug ~/Projects/maelstrom/maelstrom test -w g-set --bin ./target/debug/g_set --node-count 2 --concurrency 2n --time-limit 20 --rate 10 --log-stderr 4 | /// ``` 5 | use async_trait::async_trait; 6 | use log::debug; 7 | use maelstrom::protocol::Message; 8 | use maelstrom::{done, Node, Result, Runtime}; 9 | use serde::{Deserialize, Serialize}; 10 | use std::collections::HashSet; 11 | use std::sync::{Arc, Mutex, MutexGuard}; 12 | use std::time::Duration; 13 | 14 | pub(crate) fn main() -> Result<()> { 15 | Runtime::init(try_main()) 16 | } 17 | 18 | async fn try_main() -> Result<()> { 19 | let runtime = Runtime::new(); 20 | let handler = Arc::new(Handler::default()); 21 | runtime.with_handler(handler).run().await 22 | } 23 | 24 | #[derive(Clone, Default)] 25 | struct Handler { 26 | s: Arc>>, 27 | } 28 | 29 | #[async_trait] 30 | impl Node for Handler { 31 | async fn process(&self, runtime: Runtime, req: Message) -> Result<()> { 32 | let msg: Result = req.body.as_obj(); 33 | match msg { 34 | Ok(Request::Read {}) => { 35 | let data = to_seq(&self.s.lock().unwrap()); 36 | return runtime.reply(req, Response::ReadOk { value: data }).await; 37 | } 38 | Ok(Request::Add { element }) => { 39 | self.s.lock().unwrap().insert(element); 40 | return runtime.reply(req, Response::AddOk {}).await; 41 | } 42 | Ok(Request::ReplicateOne { element }) => { 43 | self.s.lock().unwrap().insert(element); 44 | return Ok(()); 45 | } 46 | Ok(Request::ReplicateFull { value }) => { 47 | let mut s = self.s.lock().unwrap(); 48 | for v in value { 49 | s.insert(v); 50 | } 51 | return Ok(()); 52 | } 53 | Ok(Request::Init {}) => { 54 | // spawn into tokio (instead of runtime) to not to wait 55 | // until it is completed, as it will never be. 56 | let (r0, h0) = (runtime.clone(), self.clone()); 57 | tokio::spawn(async move { 58 | loop { 59 | tokio::time::sleep(Duration::from_secs(5)).await; 60 | debug!("emit replication signal"); 61 | let s = h0.s.lock().unwrap(); 62 | for n in r0.neighbours() { 63 | let msg = Response::ReplicateFull { value: to_seq(&s) }; 64 | drop(r0.send_async(n, msg)); 65 | } 66 | } 67 | }); 68 | return Ok(()); 69 | } 70 | _ => done(runtime, req), 71 | } 72 | } 73 | } 74 | 75 | fn to_seq(s: &MutexGuard>) -> Vec { 76 | s.iter().copied().collect() 77 | } 78 | 79 | #[derive(Serialize, Deserialize)] 80 | #[serde(rename_all = "snake_case", tag = "type")] 81 | enum Request { 82 | Init {}, 83 | Read {}, 84 | Add { element: i64 }, 85 | ReplicateOne { element: i64 }, 86 | ReplicateFull { value: Vec }, 87 | } 88 | 89 | #[derive(Serialize, Deserialize)] 90 | #[serde(rename_all = "snake_case", tag = "type")] 91 | enum Response { 92 | ReadOk { value: Vec }, 93 | AddOk {}, 94 | ReplicateFull { value: Vec }, 95 | } 96 | -------------------------------------------------------------------------------- /demo/rust/src/bin/lin_kv.rs: -------------------------------------------------------------------------------- 1 | /// ```bash 2 | /// $ cargo build 3 | /// $ RUST_LOG=debug ~/Projects/maelstrom/maelstrom test -w lin-kv --bin ./target/debug/lin_kv --node-count 4 --concurrency 2n --time-limit 20 --rate 100 --log-stderr 4 | /// ```` 5 | use async_trait::async_trait; 6 | use maelstrom::kv::{lin_kv, Storage, KV}; 7 | use maelstrom::protocol::Message; 8 | use maelstrom::{done, Node, Result, Runtime}; 9 | use serde::{Deserialize, Serialize}; 10 | use std::sync::Arc; 11 | use tokio_context::context::Context; 12 | 13 | pub(crate) fn main() -> Result<()> { 14 | Runtime::init(try_main()) 15 | } 16 | 17 | async fn try_main() -> Result<()> { 18 | let runtime = Runtime::new(); 19 | let handler = Arc::new(handler(runtime.clone())); 20 | runtime.with_handler(handler).run().await 21 | } 22 | 23 | #[derive(Clone)] 24 | struct Handler { 25 | s: Storage, 26 | } 27 | 28 | #[async_trait] 29 | impl Node for Handler { 30 | async fn process(&self, runtime: Runtime, req: Message) -> Result<()> { 31 | let (ctx, _handler) = Context::new(); 32 | let msg: Result = req.body.as_obj(); 33 | match msg { 34 | Ok(Request::Read { key }) => { 35 | let value = self.s.get(ctx, key.to_string()).await?; 36 | return runtime.reply(req, Response::ReadOk { value }).await; 37 | } 38 | Ok(Request::Write { key, value }) => { 39 | self.s.put(ctx, key.to_string(), value).await?; 40 | return runtime.reply(req, Response::WriteOk {}).await; 41 | } 42 | Ok(Request::Cas { key, from, to, put }) => { 43 | self.s.cas(ctx, key.to_string(), from, to, put).await?; 44 | return runtime.reply(req, Response::CasOk {}).await; 45 | } 46 | _ => done(runtime, req), 47 | } 48 | } 49 | } 50 | 51 | fn handler(runtime: Runtime) -> Handler { 52 | Handler { s: lin_kv(runtime) } 53 | } 54 | 55 | #[derive(Serialize, Deserialize)] 56 | #[serde(rename_all = "snake_case", tag = "type")] 57 | enum Request { 58 | Read { 59 | key: u64, 60 | }, 61 | Write { 62 | key: u64, 63 | value: i64, 64 | }, 65 | Cas { 66 | key: u64, 67 | from: i64, 68 | to: i64, 69 | #[serde(default, rename = "create_if_not_exists")] 70 | put: bool, 71 | }, 72 | } 73 | 74 | #[derive(Serialize, Deserialize)] 75 | #[serde(rename_all = "snake_case", tag = "type")] 76 | enum Response { 77 | ReadOk { 78 | value: i64, 79 | }, 80 | WriteOk {}, 81 | CasOk {}, 82 | } 83 | -------------------------------------------------------------------------------- /doc/03-broadcast/broadcast-storm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jepsen-io/maelstrom/8263d1dd2b7af01dd34f5a29d0810746c2be82e2/doc/03-broadcast/broadcast-storm.png -------------------------------------------------------------------------------- /doc/03-broadcast/grid.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jepsen-io/maelstrom/8263d1dd2b7af01dd34f5a29d0810746c2be82e2/doc/03-broadcast/grid.png -------------------------------------------------------------------------------- /doc/03-broadcast/index.md: -------------------------------------------------------------------------------- 1 | # Chapter 3: Broadcast 2 | 3 | In this chapter, we generalize our [echo server](/doc/02-echo/index.md) to a 4 | re-usable Node class, and build a simple gossip-based broadcast system. 5 | We explore the impact of various network topologies on message volume and 6 | effective latency, and make our gossip protocol robust to network failures. 7 | 8 | 1. [Broadcast](01-broadcast.md) 9 | 2. [Performance](02-performance.md) 10 | -------------------------------------------------------------------------------- /doc/03-broadcast/line.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jepsen-io/maelstrom/8263d1dd2b7af01dd34f5a29d0810746c2be82e2/doc/03-broadcast/line.png -------------------------------------------------------------------------------- /doc/03-broadcast/no-comms.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jepsen-io/maelstrom/8263d1dd2b7af01dd34f5a29d0810746c2be82e2/doc/03-broadcast/no-comms.png -------------------------------------------------------------------------------- /doc/04-crdts/index.md: -------------------------------------------------------------------------------- 1 | # CRDTs 2 | 3 | Building on our [Gossip-based broadcast server](/doc/03-broadcast/index.md), we 4 | develop a family of totally-available CRDT servers. 5 | 6 | 1. [G-set](01-g-set.md) 7 | 2. [Counters](02-counters.md) 8 | -------------------------------------------------------------------------------- /doc/04-crdts/latency-partitions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jepsen-io/maelstrom/8263d1dd2b7af01dd34f5a29d0810746c2be82e2/doc/04-crdts/latency-partitions.png -------------------------------------------------------------------------------- /doc/05-datomic/g-single-realtime.svg: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 9 | 10 | a_graph 11 | 12 | 13 | 14 | T1162 15 | 16 | r 44 [1 2 3 4 5 6 7 8 9 10 11 12 13 14] 17 | 18 | r 44 [1 2 3 4 5 6 7 8 9 10 11 12 13 14] 19 | 20 | 21 | 22 | T1159 23 | 24 | a 44 15 25 | 26 | 27 | 28 | T1162:f0->T1159:f0 29 | 30 | 31 | rw 32 | 33 | 34 | 35 | T1159->T1162 36 | 37 | 38 | rt 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /doc/05-datomic/g1-realtime.svg: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 9 | 10 | a_graph 11 | 12 | 13 | 14 | T9 15 | 16 | a 9 2 17 | 18 | r 9 [2] 19 | 20 | 21 | 22 | T11 23 | 24 | a 8 4 25 | 26 | r 7 nil 27 | 28 | r 9 [2] 29 | 30 | a 9 3 31 | 32 | 33 | 34 | T9:f0->T11:f2 35 | 36 | 37 | wr 38 | 39 | 40 | 41 | T17 42 | 43 | a 4 1 44 | 45 | r 9 nil 46 | 47 | 48 | 49 | T17:f1->T9:f0 50 | 51 | 52 | rw 53 | 54 | 55 | 56 | T13 57 | 58 | a 9 4 59 | 60 | a 9 5 61 | 62 | a 9 6 63 | 64 | r 6 nil 65 | 66 | 67 | 68 | T15 69 | 70 | a 9 7 71 | 72 | 73 | 74 | T13->T15 75 | 76 | 77 | rt 78 | 79 | 80 | 81 | T11->T13 82 | 83 | 84 | rt 85 | 86 | 87 | 88 | T15->T17 89 | 90 | 91 | rt 92 | 93 | 94 | 95 | -------------------------------------------------------------------------------- /doc/05-datomic/index.md: -------------------------------------------------------------------------------- 1 | # Datomic Transactor 2 | 3 | In this chapter, we implement a strict-serializable key-value store by following [Datomic's transactor approach](https://docs.datomic.com/cloud/transactions/acid.html): data is stored in an eventually consistent key-value store, and a single mutable reference to that data is stored in a linearizable key-value store. 4 | 5 | 1. [A single-node transactor](01-single-node.md) 6 | 2. [Shared state](02-shared-state.md) 7 | 3. [Persistent Trees](03-persistent-trees.md) 8 | 4. [Optimization](04-optimization.md) 9 | -------------------------------------------------------------------------------- /doc/05-datomic/interleaved-keys.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jepsen-io/maelstrom/8263d1dd2b7af01dd34f5a29d0810746c2be82e2/doc/05-datomic/interleaved-keys.png -------------------------------------------------------------------------------- /doc/05-datomic/linear-slowdown.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jepsen-io/maelstrom/8263d1dd2b7af01dd34f5a29d0810746c2be82e2/doc/05-datomic/linear-slowdown.png -------------------------------------------------------------------------------- /doc/05-datomic/lww-high-latency.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jepsen-io/maelstrom/8263d1dd2b7af01dd34f5a29d0810746c2be82e2/doc/05-datomic/lww-high-latency.png -------------------------------------------------------------------------------- /doc/05-datomic/missing-value.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jepsen-io/maelstrom/8263d1dd2b7af01dd34f5a29d0810746c2be82e2/doc/05-datomic/missing-value.png -------------------------------------------------------------------------------- /doc/05-datomic/not-concurrent.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jepsen-io/maelstrom/8263d1dd2b7af01dd34f5a29d0810746c2be82e2/doc/05-datomic/not-concurrent.png -------------------------------------------------------------------------------- /doc/05-datomic/thunk-latency.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jepsen-io/maelstrom/8263d1dd2b7af01dd34f5a29d0810746c2be82e2/doc/05-datomic/thunk-latency.png -------------------------------------------------------------------------------- /doc/05-datomic/thunk-map.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jepsen-io/maelstrom/8263d1dd2b7af01dd34f5a29d0810746c2be82e2/doc/05-datomic/thunk-map.png -------------------------------------------------------------------------------- /doc/06-raft/final.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jepsen-io/maelstrom/8263d1dd2b7af01dd34f5a29d0810746c2be82e2/doc/06-raft/final.png -------------------------------------------------------------------------------- /doc/06-raft/index.md: -------------------------------------------------------------------------------- 1 | # Chapter 6: Raft 2 | 3 | In this chapter we write a distributed, linearizable key-value store using the 4 | Raft consensus algorithm. 5 | 6 | 1. [A Key-Value Store](01-key-value.md) 7 | 2. [Leader Election](02-leader-election.md) 8 | 3. [Replicating Logs](03-replication.md) 9 | 4. [Committing](04-committing.md) 10 | 11 | -------------------------------------------------------------------------------- /doc/06-raft/lots-of-failures.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jepsen-io/maelstrom/8263d1dd2b7af01dd34f5a29d0810746c2be82e2/doc/06-raft/lots-of-failures.png -------------------------------------------------------------------------------- /doc/06-raft/no-log-logging.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jepsen-io/maelstrom/8263d1dd2b7af01dd34f5a29d0810746c2be82e2/doc/06-raft/no-log-logging.png -------------------------------------------------------------------------------- /doc/06-raft/proxy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jepsen-io/maelstrom/8263d1dd2b7af01dd34f5a29d0810746c2be82e2/doc/06-raft/proxy.png -------------------------------------------------------------------------------- /doc/06-raft/single-node-anomaly.svg: -------------------------------------------------------------------------------- 1 | write 2read 42can't read 4 from register 2ProcessTime ─────▶LegalIllegalCrashed OpOK Op12 -------------------------------------------------------------------------------- /doc/promo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jepsen-io/maelstrom/8263d1dd2b7af01dd34f5a29d0810746c2be82e2/doc/promo.png -------------------------------------------------------------------------------- /doc/results.md: -------------------------------------------------------------------------------- 1 | # Understanding Test Results 2 | 3 | Test results are stored in `store///`. See `store/latest` 4 | for a symlink to the most recently completed test, and `store/current` for a 5 | symlink to the currently executing (or most recently completed) test. 6 | 7 | You can view these files at the command line, using a file explorer, or via 8 | Maelstrom's built-in web server. Run `java -jar maelstrom.jar serve` (or `lein 9 | run serve`) to launch the server on port 8080. 10 | 11 | ## Common Files 12 | 13 | Each [workload](workloads.md) generates slightly different files and output, 14 | but in every test, you'll find: 15 | 16 | - `jepsen.log`: The full logs from the test run, as printed to the console. 17 | 18 | - `results.edn`: The results of the test's checker, including statistics and 19 | safety analysis. This structure is also printed at the end of a test. 20 | 21 | - `history.edn`: The history of all operations performed during the test, including reads, writes, and nemesis operations. 22 | 23 | - `history.txt`: A condensed, human-readable representation of the history. 24 | Columns are `process`, `type`, `f`, `value`, and `error`. 25 | 26 | - `messages.svg`: A spacetime diagram of messages exchanged. Time flows 27 | (nonlinearly) from top to bottom, and each node is shown as a vertical bar. 28 | Messages are diagonal arrows between nodes, colorized by type. Hovering over 29 | a message shows the message itself. Helpful for understanding how your 30 | system's messages are flowing between nodes. 31 | 32 | - `timeline.html`: A diagram where time flows (nonlinearly) down, and each 33 | process's operations are arranged in a vertical track. Blue indicates `ok` 34 | operations, orange indicates indefinite (`info`) operations, and pink 35 | indicates `fail`ed operations. Helpful in understanding the concurrency 36 | structure of the test, as visible to Maelstrom. 37 | 38 | - `latency-raw.png`: Shows the latency of each Maelstrom operation, plotted 39 | over time by the time the request began. Color indicates whether the 40 | operation completed with `ok`, `info`, or `fail`. Shape indicates the `f` 41 | function for that operation: e.g. a `read`, `write`, `cas`, etc. 42 | 43 | - `latency-quantiles.png`: The same latency timeseries, binned by time, 44 | intervals and projected to quantiles 0.5, 0.95, 0.99, and 1. 45 | 46 | - `rate.png`: The overall rate of requests per second, over time, broken down 47 | by `:f` and `:type`. 48 | 49 | - `log/n*.log`: The STDERR logs emitted by each node. 50 | 51 | - `test.fressian`: A machine-readable copy of the entire test, including 52 | history and analysis. 53 | 54 | ## Interpreting Results 55 | 56 | At the end of each test, Maelstrom analyzes the history of operations using a 57 | *checker*, which produces *results*. Those results are printed to the console 58 | at the end of the test, and also written to `results.edn`. At a high level, 59 | results are a map with a single mandatory keyword, `:valid?`, which can be one 60 | of three values: 61 | 62 | - `true`: The checker considers this history legal. 63 | - `:unknown`: The checker could not determine whether the history was valid. 64 | This could happen if, for instance, the history contains no reads. 65 | - `false`: The checker identified an error of some kind. 66 | 67 | These values determine the colors of test directories in the web interface: 68 | blue means `:valid? true`, orange means `:valid? :unknown`, and pink means 69 | `:valid? false`. They also determine Jepsen's exit status when running a test: 70 | `true` exits with status 0, 1 indicates a failure, and 2 indicates an :unknown. 71 | Other exit statuses indicate internal Jepsen errors, like a crash. 72 | 73 | Maelstrom's results are a combination of several different checkers: 74 | 75 | - `perf` is always valid, and generates performance graphs like 76 | `latency-raw.png`. 77 | 78 | - `timeline` is always valid, and generates `timeline.html` 79 | 80 | - `exceptions` collects unhandled exceptions thrown during the course of a test--for example, if your binary generates malformed RPC responses. 81 | 82 | - `stats` collects basic statistics: the overall `:count` of operations, how 83 | many were `ok`, `failed`, or crashed with `info`, as well as breakdowns for 84 | each function (`by-f`), so you can see specifically how many reads vs writes 85 | failed. 86 | 87 | - `net` shows network statistics, including the overall number of sent, 88 | received, and total unique messages, and breakdowns for traffic between 89 | clients and servers vs between servers. 90 | 91 | - `workload` depends on the checker for that particular workload. See the 92 | [workload documentation](workloads.md) for details. 93 | -------------------------------------------------------------------------------- /doc/services.md: -------------------------------------------------------------------------------- 1 | # Services 2 | 3 | Services are Maelstrom-provided nodes which offer things like 'a 4 | linearizable key-value store', 'a source of sequentially-assigned 5 | timestamps', 'an eventually-consistent immutable key-value store', 'a 6 | sequentially consistent FIFO queue', and so on. Your nodes can use these as 7 | primitives for building more sophisticated systems. 8 | 9 | For instance, if you're trying to build a transactional, serializable 10 | database, you might build it as a layer on top of an existing linearizable 11 | per-key kv store--say, several distinct Raft groups, one per shard. In 12 | Maelstrom, you'd write your nodes to accept transaction requests, then (in 13 | accordance with your chosen transaction protocol) make your own key-value 14 | requests to the `lin-kv` service. 15 | 16 | To use a service, simply send an [RPC request](protocol.md) to the node ID of 17 | the service you want to use: for instance, `lin-kv`. The service will send you 18 | a response message. For an example, see [lin_kv_proxy.rb](/demo/ruby/lin_kv_proxy.rb). 19 | 20 | ## lin-kv 21 | 22 | The `lin-kv` service offers a linearizable key-value store, which has the same API as the [lin-kv workload](workloads.md#workload-lin-kv). It offers 23 | write, read, and compare-and-set operations on individual keys. 24 | 25 | Additionally, `cas` requests may include `create_if_not_exists: true`, which 26 | causes `cas` to create missing keys, rather than returning a key-not-found 27 | error. This is particularly helpful for lazy initialization of state. 28 | 29 | ## seq-kv 30 | 31 | A sequentially consistent key-value store. Just like `lin-kv`, but with relaxed 32 | consistency. 33 | 34 | All operations appear to take place in a total order. Each client 35 | observes a strictly monotonic order of operations. However, clients may 36 | interact with past states of the key-value store, provided that interaction 37 | does not violate these ordering constraints. 38 | 39 | This is *more* than simply stale reads: update operations may interact with 40 | past states, so long as doing so would not violate the total-order constraints. 41 | For example, the following non-concurrent history is legal: 42 | 43 | 1. `n1` writes x = 1 44 | 2. `n2` compare-and-sets x from 1 to 2 45 | 3. `n1` writes x = 1 46 | 4. `n2` reads x = 2 47 | 48 | This is legal because `n1`'s second write can be re-ordered to the past without 49 | violating the per-process ordering constraint, and retaining identical 50 | semantics. 51 | 52 | 1. `n1` writes x = 1 53 | 2. `n1` writes x = 1 54 | 3. `n2` compare-and-sets x from 1 to 2 55 | 4. `n2` reads x = 2 56 | 57 | ## lww-kv 58 | 59 | An intentionally pathological last-write-wins key-value store. Simulates n 60 | (default: 5) independent nodes, each of which responds to KV requests 61 | independently. Each write is assigned a roughly synchronized timestamp. Nodes 62 | periodically gossip their values and merge them together, preferring higher 63 | timestamps. The API is identical to `seq-kv` and `lin-kv`. 64 | 65 | ## lin-tso 66 | 67 | A linearizable timestamp oracle, which produces a stream of monotonically 68 | increasing integers, one for each request. This is a key component in some distributed transaction algorithms--notably, Google's Percolator. 69 | 70 | Responds to a request like: 71 | 72 | ```json 73 | { 74 | "type": "ts" 75 | } 76 | ``` 77 | 78 | with: 79 | 80 | ```json 81 | { 82 | "type": "ts_ok", 83 | "ts": 123 84 | } 85 | ``` 86 | -------------------------------------------------------------------------------- /package.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Builds the Maelstrom tarball for release. 4 | BUILD_DIR="maelstrom" 5 | TARBALL="target/maelstrom.tar.bz2" 6 | 7 | # Clean up 8 | rm -rf "$BUILD_DIR" && 9 | rm -f "$TARBALL" && 10 | 11 | # Build 12 | lein do clean, run doc, test, uberjar && 13 | 14 | # Construct directory 15 | mkdir -p "$BUILD_DIR" && 16 | mkdir "$BUILD_DIR/lib" && 17 | cp target/maelstrom-*-standalone.jar "$BUILD_DIR/lib/maelstrom.jar" && 18 | cp pkg/maelstrom "$BUILD_DIR/" && 19 | cp -r README.md "$BUILD_DIR/" && 20 | cp -r doc/ "$BUILD_DIR/" && 21 | cp -r demo "$BUILD_DIR/" && 22 | 23 | # Tar up 24 | tar cjf "$TARBALL" "$BUILD_DIR" && 25 | 26 | # Clean up 27 | rm -rf "$BUILD_DIR" 28 | -------------------------------------------------------------------------------- /pkg/maelstrom: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # A small wrapper script for invoking the Maelstrom jar, with arguments. 4 | SCRIPT_DIR=$( cd -- "$( dirname "$(readlink -f "${BASH_SOURCE[0]}")" )" &> /dev/null && pwd ) 5 | 6 | exec java -Djava.awt.headless=true -jar "${SCRIPT_DIR}/lib/maelstrom.jar" "$@" 7 | -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject maelstrom "0.2.5-SNAPSHOT" 2 | :description "A test bench for writing toy distributed systems" 3 | :url "https://github.com/jepsen-io/maelstrom" 4 | :license {:name "Eclipse Public License" 5 | :url "http://www.eclipse.org/legal/epl-v10.html"} 6 | :main maelstrom.core 7 | :jvm-opts ["-Djava.awt.headless=true" 8 | ; "-agentpath:/home/aphyr/yourkit/bin/linux-x86-64/libyjpagent.so=sampling,exceptions=disable,probe_disable=*" 9 | ] 10 | :dependencies [[org.clojure/clojure "1.12.0"] 11 | [jepsen "0.3.7"] 12 | [amalloy/ring-buffer "1.3.1"] 13 | [cheshire "5.13.0"] 14 | [byte-streams "0.2.4"] 15 | ; Reductions over journals 16 | [tesser.core "1.0.6"] 17 | [tesser.math "1.0.6"] 18 | ; For range sets 19 | [com.google.guava/guava "30.1-jre"] 20 | ; Input validation 21 | [prismatic/schema "1.4.1"] 22 | ; Random distributions 23 | [incanter/incanter-core "1.9.3"] 24 | ] 25 | :profiles {:uberjar {:aot [maelstrom.core]}}) 26 | -------------------------------------------------------------------------------- /resources/errors.edn: -------------------------------------------------------------------------------- 1 | ; This describes the various error codes that Maelstrom's workloads understand. 2 | [{:code 0 3 | :name :timeout 4 | :doc "Indicates that the requested operation could not be completed within a timeout."} 5 | {:code 1 6 | :name :node-not-found 7 | :definite? true 8 | :doc "Thrown when a client sends an RPC request to a node which does not exist."} 9 | {:code 10 10 | :name :not-supported 11 | :definite? true 12 | :doc "Use this error to indicate that a requested operation is not supported by the current implementation. Helpful for stubbing out APIs during development."} 13 | {:code 11 14 | :name :temporarily-unavailable 15 | :definite? true 16 | :doc "Indicates that the operation definitely cannot be performed at this time--perhaps because the server is in a read-only state, has not yet been initialized, believes its peers to be down, and so on. Do *not* use this error for indeterminate cases, when the operation may actually have taken place."} 17 | {:code 12 18 | :name :malformed-request 19 | :definite? true 20 | :doc "The client's request did not conform to the server's expectations, and could not possibly have been processed."} 21 | {:code 13 22 | :name :crash 23 | :doc "Indicates that some kind of general, indefinite error occurred. Use this as a catch-all for errors you can't otherwise categorize, or as a starting point for your error handler: it's safe to return `crash` for every problem by default, then add special cases for more specific errors later."} 24 | {:code 14 25 | :name :abort 26 | :definite? true 27 | :doc "Indicates that some kind of general, definite error occurred. Use this as a catch-all for errors you can't otherwise categorize, when you specifically know that the requested operation has not taken place. For instance, you might encounter an indefinite failure during the prepare phase of a transaction: since you haven't started the commit process yet, the transaction can't have taken place. It's therefore safe to return a definite `abort` to the client."} 28 | {:code 20 29 | :name :key-does-not-exist 30 | :definite? true 31 | :doc "The client requested an operation on a key which does not exist (assuming the operation should not automatically create missing keys)."} 32 | {:code 21 33 | :name :key-already-exists 34 | :definite? true 35 | :doc "The client requested the creation of a key which already exists, and the server will not overwrite it."} 36 | {:code 22 37 | :name :precondition-failed 38 | :definite? true 39 | :doc "The requested operation expected some conditions to hold, and those conditions were not met. For instance, a compare-and-set operation might assert that the value of a key is currently 5; if the value is 3, the server would return `precondition-failed`."} 40 | {:code 30 41 | :name :txn-conflict 42 | :definite? true 43 | :doc "The requested transaction has been aborted because of a conflict with another transaction. Servers need not return this error on every conflict: they may choose to retry automatically instead."} 44 | ] 45 | -------------------------------------------------------------------------------- /resources/protocol-intro.md: -------------------------------------------------------------------------------- 1 | # Protocol 2 | 3 | Maelstrom nodes receive messages on STDIN, send messages on STDOUT, and log 4 | debugging output on STDERR. Maelstrom nodes must not print anything that is not 5 | a message to STDOUT. Maelstrom will log STDERR output to disk for you. 6 | 7 | ## Nodes and Networks 8 | 9 | A Maelstrom test simulates a distributed system by running many *nodes*, and a 10 | network which routes *messages* between them. Each node has a unique string 11 | identifier, used to route messages to and from that node. 12 | 13 | - Nodes `n1`, `n2`, `n3`, etc. are instances of the binary you pass to 14 | Maelstrom. These nodes implement whatever distributed algorithm you're trying 15 | to build: for instance, a key-value store. You can think of these as 16 | *servers*, in that they accept requests from clients and send back responses. 17 | 18 | - Nodes `c1`, `c2`, `c3`, etc. are Maelstrom's internal clients. Clients send 19 | requests to servers and expect responses back, via a simple asynchronous [RPC 20 | protocol](#message-bodies). 21 | 22 | ## Messages 23 | 24 | Both STDIN and STDOUT messages are JSON objects, separated by newlines (`\n`). Each message object is of the form: 25 | 26 | ```edn 27 | { 28 | "src": A string identifying the node this message came from 29 | "dest": A string identifying the node this message is to 30 | "body": An object: the payload of the message 31 | } 32 | ``` 33 | 34 | ## Message Bodies 35 | 36 | RPC messages exchanged with Maelstrom's clients have bodies with the following 37 | reserved keys: 38 | 39 | ```edn 40 | { 41 | "type": (mandatory) A string identifying the type of message this is 42 | "msg_id": (optional) A unique integer identifier 43 | "in_reply_to": (optional) For req/response, the msg_id of the request 44 | } 45 | ``` 46 | 47 | Message IDs should be unique on the node which sent them. For instance, each 48 | node can use a monotonically increasing integer as their source of message IDs. 49 | 50 | Each message has additional keys, depending on what kind of message it is. For 51 | example, here is a read request from the `lin_kv` workload, which asks for the 52 | current value of key `3`: 53 | 54 | ```json 55 | { 56 | "type": "read", 57 | "msg_id": 123, 58 | "key": 3 59 | } 60 | ``` 61 | 62 | And its corresponding response, indicating the value is presently `4`: 63 | 64 | ```json 65 | { 66 | "type": "read_ok", 67 | "msg_id": 56, 68 | "in_reply_to": 123, 69 | "value": 4 70 | } 71 | ``` 72 | 73 | The various message types and the meanings of their fields are defined in the 74 | [workload documentation](workloads.md). 75 | 76 | Messages exchanged between your server nodes may have any `body` structure you 77 | like; you are not limited to request-response, and may invent any message 78 | semantics you choose. If some of your messages *do* use the body format 79 | described above, Maelstrom can help generate useful visualizations and 80 | statistics for those messages. 81 | 82 | ## Initialization 83 | 84 | At the start of a test, Maelstrom issues a single `init` message to each node, 85 | like so: 86 | 87 | ```json 88 | { 89 | "type": "init", 90 | "msg_id": 1, 91 | "node_id": "n3", 92 | "node_ids": ["n1", "n2", "n3"] 93 | } 94 | ``` 95 | 96 | The `node_id` field indicates the ID of the node which is receiving this 97 | message: here, the node ID is "n3". Your node should remember this ID and 98 | include it as the `src` of any message it sends. 99 | 100 | The `node_ids` field lists all nodes in the cluster, including the recipient. 101 | All nodes receive an identical list; you may use its order if you like. 102 | 103 | In response to the `init` message, each node must respond with a message of 104 | type `init_ok`. 105 | 106 | ```json 107 | { 108 | "type": "init_ok", 109 | "in_reply_to": 1 110 | } 111 | ``` 112 | 113 | ## Errors 114 | 115 | In response to a Maelstrom RPC request, a node may respond with an *error* 116 | message, whose `body` is a JSON object like so: 117 | 118 | ```json 119 | { 120 | "type": "error", 121 | "in_reply_to": 5, 122 | "code": 11, 123 | "text": "Node n5 is waiting for quorum and cannot service requests yet" 124 | } 125 | ``` 126 | 127 | The `type` of an error body is always `\"error\"`. 128 | 129 | As with all RPC responses, the `in_reply_to` field is the `msg_id` of 130 | the request which caused this error. 131 | 132 | The `code` is an integer which indicates the type of error which occurred. 133 | Maelstrom defines several error types, and you can also invent your own. 134 | Codes 0-999 are reserved for Maelstrom's use; codes 1000 and above are free 135 | for your own purposes. 136 | 137 | The `text` field is a free-form string. It is optional, and may contain any 138 | explanatory message you like. 139 | 140 | You may include other keys in the error body, if you like; Maelstrom will 141 | retain them as a part of the history, and they may be helpful in your own 142 | analysis. 143 | 144 | Errors are either *definite* or *indefinite*. A definite error means that the 145 | requested operation definitely did not (and never will) happen. An indefinite 146 | error means that the operation might have happened, or might never happen, or 147 | might happen at some later time. Maelstrom uses this information to interpret 148 | histories correctly, so it's important that you never return a definite error 149 | under indefinite conditions. When in doubt, indefinite is always safe. Custom 150 | error codes are always indefinite. 151 | 152 | The following table lists all of Maelstrom's defined errors. 153 | 154 | -------------------------------------------------------------------------------- /resources/workloads-intro.md: -------------------------------------------------------------------------------- 1 | # Workloads 2 | 3 | A *workload* specifies the semantics of a distributed system: what 4 | operations are performed, how clients submit requests to the system, what 5 | those requests mean, what kind of responses are expected, which errors can 6 | occur, and how to check the resulting history for safety. 7 | 8 | For instance, the *broadcast* workload says that clients submit `broadcast` 9 | messages to arbitrary servers, and can send a `read` request to obtain the 10 | set of all broadcasted messages. Clients mix reads and broadcast operations 11 | throughout the history, and at the end of the test, perform a final read 12 | after allowing a brief period for convergence. To check broadcast histories, 13 | Maelstrom looks to see how long it took for messages to be broadcast, and 14 | whether any were lost. 15 | 16 | This is a reference document, automatically generated from Maelstrom's source 17 | code by running `lein run doc`. For each workload, it describes the general 18 | semantics of that workload, what errors are allowed, and the structure of RPC 19 | messages that you'll need to handle. 20 | -------------------------------------------------------------------------------- /src/maelstrom/checker.clj: -------------------------------------------------------------------------------- 1 | (ns maelstrom.checker 2 | "Common Maelstrom checkers" 3 | (:require [jepsen [checker :as checker] 4 | [history :as h]])) 5 | 6 | (defn availability-checker 7 | "Checks to see whether the history met the test's target :availability goals. 8 | There are several possible values for availability: 9 | 10 | 0 nil, which imposes no availability checks 11 | 1. :total, which means every request must succeed. 12 | 2. A number between 0 and 1, which is a lower bound on the fraction of 13 | operations that must succeed." 14 | [] 15 | (reify checker/Checker 16 | (check [_ test history opts] 17 | (let [a (:availability test) 18 | ok-count (count (h/oks history)) 19 | invoke-count (count (h/invokes history)) 20 | res {:valid? true 21 | ;:invoke-count invoke-count 22 | ;:ok-count ok-count 23 | :ok-fraction (if (zero? invoke-count) 24 | 1 25 | (float (/ ok-count invoke-count)))}] 26 | (cond 27 | (nil? a) 28 | res 29 | 30 | (= :total a) 31 | (assoc res :valid? (== 1 (:ok-fraction res))) 32 | 33 | (number? a) 34 | (assoc res :valid? (<= a (:ok-fraction res))) 35 | 36 | true 37 | (throw (IllegalArgumentException. 38 | (str "Don't know how to handle :availability " 39 | (pr-str a))))))))) 40 | -------------------------------------------------------------------------------- /src/maelstrom/db.clj: -------------------------------------------------------------------------------- 1 | (ns maelstrom.db 2 | "Shared functionality for starting database 'nodes'" 3 | (:require [clojure.tools.logging :refer [info warn]] 4 | [jepsen [core :as jepsen] 5 | [db :as db] 6 | [store :as store]] 7 | [maelstrom [client :as client] 8 | [net :as net] 9 | [process :as process] 10 | [service :as service]] 11 | [slingshot.slingshot :refer [try+ throw+]])) 12 | 13 | (defn db 14 | "Options: 15 | 16 | :bin - a binary to run 17 | :args - args to that binary 18 | :net - a network" 19 | [opts] 20 | (let [net (:net opts) 21 | services (atom nil) 22 | processes (atom {})] 23 | (reify db/DB 24 | (setup! [_ test node-id] 25 | ; Spawn built-in Maelstrom services 26 | (when (= (jepsen/primary test) node-id) 27 | (reset! services (service/start-services! 28 | net 29 | (service/default-services test)))) 30 | 31 | ; Start this node 32 | (info "Setting up" node-id) 33 | (swap! processes assoc node-id 34 | (process/start-node! 35 | {:node-id node-id 36 | :bin (:bin opts) 37 | :args (:args opts) 38 | :net net 39 | :dir (System/getProperty "java.io.tmpdir") 40 | :log-stderr? (:log-stderr test) 41 | :log-file (->> (str node-id ".log") 42 | (store/path test "node-logs") 43 | .getCanonicalPath)})) 44 | 45 | ; Initialize this node 46 | (let [client (client/open! net)] 47 | (try+ 48 | (let [res (client/rpc! 49 | client 50 | node-id 51 | {:type "init" 52 | :node_id node-id 53 | :node_ids (:nodes test)} 54 | 10000)] 55 | (when (not= "init_ok" (:type res)) 56 | (throw+ {:type :init-failed 57 | :node node-id 58 | :response res} 59 | nil 60 | (str "Expected an init_ok message, but node responded with " 61 | (pr-str res))))) 62 | (catch [:type :maelstrom.client/timeout] e 63 | (throw+ {:type :init-failed 64 | :node node-id} 65 | (:throwable &throw-context) 66 | (str "Expected node " node-id 67 | " to respond to an init message, but node did not respond."))) 68 | (finally 69 | (client/close! client))))) 70 | 71 | (teardown! [_ test node] 72 | ; Tear down node 73 | (when-let [p (get @processes node)] 74 | (info "Tearing down" node) 75 | (process/stop-node! p) 76 | (swap! processes dissoc node)) 77 | 78 | ; Tear down services 79 | (when (= node (jepsen/primary test)) 80 | (when-let [s @services] 81 | (service/stop-services! s) 82 | (reset! services nil))))))) 83 | -------------------------------------------------------------------------------- /src/maelstrom/doc.clj: -------------------------------------------------------------------------------- 1 | (ns maelstrom.doc 2 | "Generates documentation files from Maelstrom's internal registries of RPC 3 | operations and errors." 4 | (:require [clojure [pprint :refer [pprint]] 5 | [string :as str]] 6 | [clojure.java.io :as io] 7 | [clojure.tools.logging :refer [info warn]] 8 | [maelstrom [client :as c]])) 9 | 10 | (def workloads-filename 11 | "Where should we write workloads.md?" 12 | "doc/workloads.md") 13 | 14 | (def protocol-filename 15 | "Where should we write protocol.md?" 16 | "doc/protocol.md") 17 | 18 | (defn unindent 19 | "Strips leading whitespace from all lines in a string." 20 | [s] 21 | (str/replace s #"(^|\n)[ \t]+" "$1")) 22 | 23 | (defn print-workloads 24 | "Prints out all workloads to stdout, based on the client RPC registry." 25 | ([] 26 | (print-workloads @c/rpc-registry)) 27 | ([rpcs] 28 | ; Group RPCs by namespace 29 | (let [ns->rpcs (->> rpcs 30 | (group-by (fn [rpc] 31 | (-> rpc 32 | :ns 33 | ns-name 34 | name 35 | (str/split #"\.") 36 | last))) 37 | (sort-by key))] 38 | 39 | (println (slurp (io/resource "workloads-intro.md")) "\n") 40 | 41 | (println "## Table of Contents\n") 42 | (doseq [[ns rpcs] ns->rpcs] 43 | (println (str "- [" (str/capitalize ns) "](#workload-" ns ")"))) 44 | (println) 45 | 46 | (doseq [[ns rpcs] ns->rpcs] 47 | (println "## Workload:" (str/capitalize ns) "\n") 48 | 49 | (println (unindent (:doc (meta (:ns (first rpcs))))) "\n") 50 | 51 | (doseq [rpc rpcs] 52 | (println "### RPC:" (str/capitalize (:name rpc)) "\n") 53 | (println (unindent (:doc rpc)) "\n") 54 | (println "Request:\n") 55 | (println "```clj") 56 | (pprint (:send rpc)) 57 | (println "```") 58 | (println "\nResponse:\n") 59 | (println "```clj") 60 | (pprint (:recv rpc)) 61 | (println "```") 62 | (println "\n")) 63 | 64 | (println))))) 65 | 66 | (defn print-error-registry 67 | "Prints out the error registry, as Markdown, for documentation purposes." 68 | [] 69 | (assert (seq @c/error-registry) 70 | "Error registry empty, maybe macroexpansion cached? Try `lein clean`?") 71 | 72 | (println "| Code | Name | Definite | Description |") 73 | (println "| ---: | :--- | :------: | :---------- |") 74 | 75 | (doseq [{:keys [code name definite? doc]} (map val (sort @c/error-registry))] 76 | (println "|" code "|" 77 | (clojure.core/name name) "|" 78 | (if definite? "✓" " ") "|" 79 | (str/replace doc #"\n" " ") "|"))) 80 | 81 | (defn print-protocol 82 | "Prints out the protocol documentation, including errors." 83 | [] 84 | (println (slurp (io/resource "protocol-intro.md"))) 85 | (print-error-registry)) 86 | 87 | (defn write-docs! 88 | "Writes out all documentation files." 89 | [] 90 | (with-open [w (io/writer workloads-filename)] 91 | (binding [*out* w] 92 | (print-workloads))) 93 | 94 | (with-open [w (io/writer protocol-filename)] 95 | (binding [*out* w] 96 | (print-protocol)))) 97 | -------------------------------------------------------------------------------- /src/maelstrom/nemesis.clj: -------------------------------------------------------------------------------- 1 | (ns maelstrom.nemesis 2 | "Fault injection" 3 | (:require [clojure.tools.logging :refer [info warn]] 4 | [jepsen [generator :as gen] 5 | [nemesis :as n] 6 | [util :refer [pprint-str]]] 7 | [jepsen.nemesis.combined :as nc] 8 | [slingshot.slingshot :refer [try+ throw+]])) 9 | 10 | (defn package 11 | "A full nemesis package. Options are those for 12 | jepsen.nemesis.combined/nemesis-package." 13 | [opts] 14 | (nc/compose-packages 15 | [(nc/partition-package opts) 16 | (nc/db-package opts)])) 17 | -------------------------------------------------------------------------------- /src/maelstrom/net/checker.clj: -------------------------------------------------------------------------------- 1 | (ns maelstrom.net.checker 2 | "This checker uses net.journal and net.viz to generate statistics and 3 | visualizations from the Fressian journal files in a store dir." 4 | (:require [clojure.tools.logging :refer [info warn]] 5 | [fipp.edn :refer [pprint]] 6 | [jepsen [checker :as checker] 7 | [store :as store] 8 | [util :as util :refer [linear-time-nanos 9 | nanos->ms 10 | ms->nanos]]] 11 | [maelstrom.util :as u] 12 | [maelstrom.net [journal :as j] 13 | [message :as msg] 14 | [viz :as viz]] 15 | [tesser [core :as t] 16 | [math :as tm] 17 | [utils :as tu]]) 18 | (:import (java.io Closeable 19 | File 20 | EOFException) 21 | (java.util BitSet) 22 | (maelstrom.net.message Message) 23 | (io.lacuna.bifurcan ISet 24 | LinearSet 25 | Maps 26 | Set))) 27 | 28 | (defn basic-stats 29 | "A fold for aggregate statistics over a journal." 30 | [journal] 31 | (->> journal 32 | (t/fuse {:send-count (t/count j/sends) 33 | :recv-count (t/count j/recvs) 34 | :msg-count (->> (t/map (comp :id :message)) 35 | ; (fast-cardinality))}))) 36 | (j/dense-int-cardinality))}))) 37 | 38 | (def stats 39 | (t/fuse {:all (->> (t/map identity) basic-stats) 40 | :clients (->> j/clients basic-stats) 41 | :servers (->> j/servers basic-stats)})) 42 | 43 | (defn checker 44 | "A Jepsen checker which extracts the journal and analyzes its statistics." 45 | [] 46 | (reify checker/Checker 47 | (check [this test history opts] 48 | (let [; Fire off the plotter immediately; it can run without us 49 | plot (future (viz/plot-analemma! test)) 50 | ; Compute stats 51 | stats (->> stats 52 | (j/tesser-journal test)) 53 | ; Add msgs-per-op stats, so we can tell roughly how many messages 54 | ; exchanged per logical operation 55 | op-count (->> history 56 | (remove (comp #{:nemesis} :process)) 57 | (filter (comp #{:invoke} :type)) 58 | count) 59 | stats (if (zero? op-count) 60 | stats 61 | (-> stats 62 | (assoc-in [:all :msgs-per-op] 63 | (float (/ (:msg-count (:all stats)) 64 | op-count))) 65 | (assoc-in [:servers :msgs-per-op] 66 | (float (/ (:msg-count (:servers stats)) 67 | op-count)))))] 68 | ; Block on plot 69 | @plot 70 | (assoc stats :valid? true))))) 71 | -------------------------------------------------------------------------------- /src/maelstrom/net/message.clj: -------------------------------------------------------------------------------- 1 | (ns maelstrom.net.message 2 | "Contains operations specifically related to network messages; used by both 3 | net and net.journal." 4 | (:require [schema.core :as s])) 5 | 6 | 7 | 8 | (defrecord Message [^long id src dest body]) 9 | 10 | (defn message 11 | "Constructs a new Message. If no ID is provided, uses -1." 12 | ([src dest body] 13 | (Message. -1 src dest body)) 14 | ([id src dest body] 15 | (Message. id src dest body))) 16 | 17 | (defn validate 18 | "Checks to make sure a message is well-formed. Returns msg if legal, 19 | otherwise throws." 20 | [m] 21 | (assert (instance? Message m) 22 | (str "Expected message " (pr-str m) " to be a Message")) 23 | (assert (:src m) (str "No source for message " (pr-str m))) 24 | (assert (:dest m) (str "No destination for message " (pr-str m))) 25 | m) 26 | -------------------------------------------------------------------------------- /src/maelstrom/util.clj: -------------------------------------------------------------------------------- 1 | (ns maelstrom.util 2 | "Kitchen sink" 3 | (:require [schema.core :as s] 4 | [clojure.string :as str] 5 | [slingshot.slingshot :refer [try+ throw+]])) 6 | 7 | (defn client? 8 | "Is a given node id a client?" 9 | [^String node-id] 10 | (= \c (.charAt node-id 0))) 11 | 12 | (defn involves-client? 13 | "Does a given network message involve a client?" 14 | [message] 15 | (or (client? (:src message)) 16 | (client? (:dest message)))) 17 | 18 | (defn sort-clients 19 | "Sorts a collection by client ID. We split up the letter and number parts, to 20 | give a nice numeric order." 21 | [clients] 22 | (->> clients 23 | (sort-by (fn [client] 24 | (if-let [[_ type num] (re-find #"(\w+?)(\d+)" client)] 25 | ; Typical 'c1', 'n4', etc 26 | [0 type (Long/parseLong num)] 27 | ; Sort special (services) nodes last 28 | [1 client 0]))))) 29 | 30 | (defn map->pairs 31 | "Encodes a map {k v, k2 v2} as KV pairs [[k v] [k2 v2]], for JSON 32 | serialization." 33 | [m] 34 | (seq m)) 35 | 36 | (defn pairs->map 37 | "Decodes a sequence of kv pairs [[k v] [k2 v2]] to a map {k v, k2 v2}, for 38 | JSON deserialization." 39 | [pairs] 40 | (into {} pairs)) 41 | -------------------------------------------------------------------------------- /src/maelstrom/workload/echo.clj: -------------------------------------------------------------------------------- 1 | (ns maelstrom.workload.echo 2 | "A simple echo workload: sends a message, and expects to get that same 3 | message back." 4 | (:require [maelstrom [client :as c] 5 | [net :as net]] 6 | [jepsen [checker :as checker] 7 | [client :as client] 8 | [generator :as gen] 9 | [independent :as independent]] 10 | [jepsen.tests.linearizable-register :as lin-reg] 11 | [knossos.history :as history] 12 | [schema.core :as s] 13 | [slingshot.slingshot :refer [try+ throw+]])) 14 | 15 | (c/defrpc echo! 16 | "Clients send `echo` messages to servers with an `echo` field containing an 17 | arbitrary payload they'd like to have sent back. Servers should respond with 18 | `echo_ok` messages containing that same payload." 19 | {:type (s/eq "echo") 20 | :echo s/Any} 21 | {:type (s/eq "echo_ok") 22 | :echo s/Any}) 23 | 24 | (defn client 25 | ([net] 26 | (client net nil nil)) 27 | ([net conn node] 28 | (reify client/Client 29 | (open! [this test node] 30 | (client net (c/open! net) node)) 31 | 32 | (setup! [this test]) 33 | 34 | (invoke! [_ test op] 35 | (c/with-errors op #{} 36 | (try+ (let [res (echo! conn node {:echo (:value op)})] 37 | (assoc op :type :ok, :value res))))) 38 | 39 | (teardown! [_ test]) 40 | 41 | (close! [_ test] 42 | (c/close! conn))))) 43 | 44 | (defn checker 45 | "Expects responses to every echo operation to match the invocation's value." 46 | [] 47 | (reify checker/Checker 48 | (check [this test history opts] 49 | (let [pairs (history/pair-index history) 50 | errs (keep (fn [[invoke complete]] 51 | (cond ; Only take invoke/complete pairs 52 | (not= (:type invoke) :invoke) 53 | nil 54 | 55 | (not= (:value invoke) 56 | (:echo (:value complete))) 57 | ["Expected a message with :echo" 58 | (:value invoke) 59 | "But received" 60 | (:value complete)])) 61 | pairs)] 62 | {:valid? (empty? errs) 63 | :errors errs})))) 64 | 65 | (defn workload 66 | "Constructs a workload for linearizable registers, given option from the CLI 67 | test constructor: 68 | 69 | {:net A Maelstrom network}" 70 | [opts] 71 | {:client (client (:net opts)) 72 | :generator (->> (fn [] 73 | {:f :echo 74 | :value (str "Please echo " (rand-int 128))}) 75 | (gen/each-thread)) 76 | :checker (checker)}) 77 | -------------------------------------------------------------------------------- /src/maelstrom/workload/g_counter.clj: -------------------------------------------------------------------------------- 1 | (ns maelstrom.workload.g-counter 2 | "An eventually-consistent grow-only counter, which supports increments. 3 | Validates that the final read on each node has a value which is the sum of 4 | all known (or possible) increments. 5 | 6 | See also: pn-counter, which is identical, but allows decrements." 7 | (:refer-clojure :exclude [read]) 8 | (:require [jepsen.generator :as gen] 9 | [schema.core :as s] 10 | [maelstrom [client :as c]] 11 | [maelstrom.workload.pn-counter :as pn-counter])) 12 | 13 | ; These are only here for docs, and actually don't get used--see their 14 | ; counterparts in pn-counter. 15 | (c/defrpc add! 16 | "Adds a non-negative integer, called `delta`, to the counter. Servers should 17 | respond with an `add_ok` message." 18 | {:type (s/eq "add") 19 | :delta s/Int} 20 | {:type (s/eq "add_ok")}) 21 | 22 | (c/defrpc read 23 | "Reads the current value of the counter. Servers respond with a `read_ok` 24 | message containing a `value`, which should be the sum of all (known) added 25 | deltas." 26 | {:type (s/eq "read")} 27 | {:type (s/eq "read_ok") 28 | :value s/Int}) 29 | 30 | (defn workload 31 | "Constructs a workload for a grow-only counter, given options from the CLI 32 | test constructor: 33 | 34 | {:net A Maelstrom network}" 35 | [opts] 36 | (update (pn-counter/workload opts) 37 | :generator 38 | (partial gen/filter (fn [op] 39 | (not (and (= :add (:f op)) 40 | (neg? (:value op)))))))) 41 | -------------------------------------------------------------------------------- /src/maelstrom/workload/g_set.clj: -------------------------------------------------------------------------------- 1 | (ns maelstrom.workload.g-set 2 | "A grow-only set workload: clients add elements to a set, and read the 3 | current value of the set." 4 | (:refer-clojure :exclude [read]) 5 | (:require [maelstrom [client :as c] 6 | [net :as net]] 7 | [jepsen [checker :as checker] 8 | [client :as client] 9 | [generator :as gen]] 10 | [schema.core :as s] 11 | [slingshot.slingshot :refer [try+ throw+]])) 12 | 13 | (c/defrpc add! 14 | "Requests that a server add a single element to the set. Acknowledged by an 15 | `add_ok` message." 16 | {:type (s/eq "add") 17 | :element s/Any} 18 | {:type (s/eq "add_ok")}) 19 | 20 | (c/defrpc read 21 | "Requests the current set of all elements. Servers respond with a message 22 | containing an `elements` key, whose `value` is a JSON array of added 23 | elements." 24 | {:type (s/eq "read")} 25 | {:type (s/eq "read_ok") 26 | :value [s/Any]}) 27 | 28 | (defn client 29 | ([net] 30 | (client net nil nil)) 31 | ([net conn node] 32 | (reify client/Client 33 | (open! [this test node] 34 | (client net (c/open! net) node)) 35 | 36 | (setup! [this test]) 37 | 38 | (invoke! [_ test op] 39 | (case (:f op) 40 | :add (do (add! conn node {:element (:value op)}) 41 | (assoc op :type :ok)) 42 | 43 | :read (assoc op 44 | :type :ok 45 | :value (:value (read conn node {}))))) 46 | 47 | (teardown! [_ test]) 48 | 49 | (close! [_ test] 50 | (c/close! conn))))) 51 | 52 | (defn workload 53 | "Constructs a workload for a grow-only set, given options from the CLI 54 | test constructor: 55 | 56 | {:net A Maelstrom network}" 57 | [opts] 58 | {:client (client (:net opts)) 59 | :generator (gen/mix [(->> (range) (map (fn [x] {:f :add, :value x}))) 60 | (repeat {:f :read})]) 61 | :final-generator (gen/each-thread {:f :read}) 62 | :checker (checker/set-full)}) 63 | -------------------------------------------------------------------------------- /src/maelstrom/workload/lin_kv.clj: -------------------------------------------------------------------------------- 1 | (ns maelstrom.workload.lin-kv 2 | "A workload for a linearizable key-value store." 3 | (:refer-clojure :exclude [read]) 4 | (:require [maelstrom [client :as c] 5 | [net :as net]] 6 | [jepsen [client :as client] 7 | [generator :as gen] 8 | [independent :as independent]] 9 | [jepsen.tests.linearizable-register :as lin-reg] 10 | [schema.core :as s])) 11 | 12 | (c/defrpc read 13 | "Reads the current value of a single key. Clients send a `read` request with 14 | the key they'd like to observe, and expect a response with the current 15 | `value` of that key." 16 | {:type (s/eq "read") 17 | :key s/Any} 18 | {:type (s/eq "read_ok") 19 | :value s/Any}) 20 | 21 | (c/defrpc write! 22 | "Blindly overwrites the value of a key. Creates keys if they do not presently 23 | exist. Servers should respond with a `read_ok` response once the write is 24 | complete." 25 | {:type (s/eq "write") 26 | :key s/Any 27 | :value s/Any} 28 | {:type (s/eq "write_ok")}) 29 | 30 | (c/defrpc cas! 31 | "Atomically compare-and-sets a single key: if the value of `key` is currently 32 | `from`, sets it to `to`. Returns error 20 if the key doesn't exist, and 22 if 33 | the `from` value doesn't match." 34 | {:type (s/eq "cas") 35 | :key s/Any 36 | :from s/Any 37 | :to s/Any} 38 | {:type (s/eq "cas_ok")}) 39 | 40 | (defn client 41 | "Construct a linearizable key-value client for the given network" 42 | ([net] 43 | (client net nil nil)) 44 | ([net conn node] 45 | (reify client/Client 46 | (open! [this test node] 47 | (client net (c/open! net) node)) 48 | 49 | (setup! [this test]) 50 | 51 | (invoke! [_ test op] 52 | (c/with-errors op #{:read} 53 | (let [[k v] (:value op) 54 | timeout (max (* 10 (:mean (:latency test))) 1000)] 55 | (case (:f op) 56 | :read (let [res (read conn node {:key k} timeout) 57 | v (:value res)] 58 | (assoc op 59 | :type :ok 60 | :value (independent/tuple k v))) 61 | 62 | :write (let [res (write! conn node {:key k, :value v} timeout)] 63 | (assoc op :type :ok)) 64 | 65 | :cas (let [[v v'] v 66 | res (cas! conn node {:key k, :from v, :to v'} timeout)] 67 | (assoc op :type :ok)))))) 68 | 69 | (teardown! [_ test]) 70 | 71 | (close! [_ test] 72 | (c/close! conn)) 73 | 74 | client/Reusable 75 | (reusable? [this test] 76 | true)))) 77 | 78 | (defn workload 79 | "Constructs a workload for linearizable registers, given option from the CLI 80 | test constructor: 81 | 82 | {:net A Maelstrom network}" 83 | [opts] 84 | (-> (lin-reg/test {:nodes (:nodes opts)}) 85 | (assoc :client (client (:net opts))))) 86 | -------------------------------------------------------------------------------- /src/maelstrom/workload/pn_counter.clj: -------------------------------------------------------------------------------- 1 | (ns maelstrom.workload.pn-counter 2 | "An eventually-consistent counter which supports increments and decrements. 3 | Validates that the final read on each node has a value which is the sum of 4 | all known (or possible) increments and decrements. 5 | 6 | See also: g-counter, which is identical, but does not allow decrements." 7 | (:refer-clojure :exclude [read]) 8 | (:require [maelstrom [client :as c] 9 | [net :as net]] 10 | [jepsen [checker :as checker] 11 | [client :as client] 12 | [generator :as gen]] 13 | [knossos.op :as op] 14 | [schema.core :as s] 15 | [slingshot.slingshot :refer [try+ throw+]]) 16 | (:import (com.google.common.collect Range 17 | RangeSet 18 | TreeRangeSet))) 19 | 20 | (c/defrpc add! 21 | "Adds a (potentially negative) integer, called `delta`, to the counter. 22 | Servers should respond with an `add_ok` message." 23 | {:type (s/eq "add") 24 | :delta s/Int} 25 | {:type (s/eq "add_ok")}) 26 | 27 | (c/defrpc read 28 | "Reads the current value of the counter. Servers respond with a `read_ok` 29 | message containing a `value`, which should be the sum of all (known) added 30 | deltas." 31 | {:type (s/eq "read")} 32 | {:type (s/eq "read_ok") 33 | :value s/Int}) 34 | 35 | (defn client 36 | ([net] 37 | (client net nil nil)) 38 | ([net conn node] 39 | (reify client/Client 40 | (open! [this test node] 41 | (client net (c/open! net) node)) 42 | 43 | (setup! [this test]) 44 | 45 | (invoke! [_ test op] 46 | (case (:f op) 47 | :add (do (add! conn node {:delta (:value op)}) 48 | (assoc op :type :ok)) 49 | 50 | :read (->> (read conn node {}) 51 | :value 52 | long 53 | (assoc op :type :ok, :value)))) 54 | 55 | (teardown! [_ test]) 56 | 57 | (close! [_ test] 58 | (c/close! conn))))) 59 | 60 | (defn range->vec 61 | "Converts an open range into a closed integer [lower upper] pair." 62 | [^Range r] 63 | [(inc (.lowerEndpoint r)) 64 | (dec (.upperEndpoint r))]) 65 | 66 | (defn acceptable->vecs 67 | "Turns an acceptable TreeRangeSet into a vector of [lower upper] inclusive 68 | ranges." 69 | [^TreeRangeSet s] 70 | (map range->vec (.asRanges s))) 71 | 72 | (defn acceptable-range 73 | "Takes a lower and upper bound for a range and constructs a Range for an 74 | acceptable TreeRangeSet. The constructed range will be an *open* range from 75 | lower - 1 to upper + 1, which ensures that merges work correctly." 76 | [lower upper] 77 | (Range/open (dec lower) (inc upper))) 78 | 79 | (defn checker 80 | "This checker verifies that every final read is the sum of all 81 | known-completed adds plus any number of possibly-completed adds. Returns a 82 | map with :valid? true if all reads marked :final? are in the acceptable set. 83 | Returns the acceptable set, encoded as a sequence of [lower upper] closed 84 | ranges." 85 | [] 86 | (reify checker/Checker 87 | (check [this test history opts] 88 | (let [; First, let's get all the add operations 89 | adds (filter (comp #{:add} :f) history) 90 | ; What's the total of the ops we *definitely* know happened? 91 | definite-sum (->> adds 92 | (filter op/ok?) 93 | (map :value) 94 | (reduce +)) 95 | ; What are all the possible outcomes of indeterminate ops? 96 | acceptable (TreeRangeSet/create) 97 | _ (.add acceptable (acceptable-range definite-sum definite-sum)) 98 | ; For each possible add, we want to allow that either to happen or 99 | ; not. 100 | _ (doseq [add adds] 101 | (when (op/info? add) 102 | (let [delta (:value add)] 103 | ; For each range, add delta, and merge that back in. Note 104 | ; we materialize asRanges to avoid iterating during our 105 | ; mutation. 106 | (doseq [^Range r (vec (.asRanges acceptable))] 107 | (.add acceptable 108 | (Range/open (+ (.lowerEndpoint r) delta) 109 | (+ (.upperEndpoint r) delta))))))) 110 | ; Now, extract the final reads for each node 111 | reads (->> history 112 | (filter :final?) 113 | (filter op/ok?)) 114 | ; And find any problems 115 | errors (->> reads 116 | (filter (fn [r] 117 | ; If we get a fractional read for some 118 | ; reason, our cute open-range technique is 119 | ; gonna give wrong answers 120 | (assert (integer? (:value r))) 121 | (not (.contains acceptable (:value r))))))] 122 | {:valid? (empty? errors) 123 | :errors (seq errors) 124 | :final-reads (map :value reads) 125 | :acceptable (acceptable->vecs acceptable)})))) 126 | 127 | (defn workload 128 | "Constructs a workload for a grow-only set, given options from the CLI 129 | test constructor: 130 | 131 | {:net A Maelstrom network}" 132 | [opts] 133 | {:client (client (:net opts)) 134 | :generator (gen/mix [(fn [] {:f :add, :value (- (rand-int 10) 5)}) 135 | (repeat {:f :read})]) 136 | :final-generator (gen/each-thread {:f :read, :final? true}) 137 | :checker (checker)}) 138 | -------------------------------------------------------------------------------- /src/maelstrom/workload/txn_list_append.clj: -------------------------------------------------------------------------------- 1 | (ns maelstrom.workload.txn-list-append 2 | "A transactional workload over a map of keys to lists of elements. Clients 3 | submit a single transaction via a `txn` request, and expect a 4 | completed version of that transaction in a `txn_ok` response. 5 | 6 | A transaction is an array of micro-operations, which should be executed in 7 | order: 8 | 9 | ```edn 10 | [op1, op2, ...] 11 | ``` 12 | 13 | Each micro-op is a 3-element array comprising a function, key, and value: 14 | 15 | ```edn 16 | [f, k, v] 17 | ``` 18 | 19 | There are two functions. A *read* observes the current value of a specific 20 | key. `[\"r\", 5, [1, 2]]` denotes that a read of key 5 observed the list `[1, 21 | 2]`. When clients submit reads, they leave their values `null`: `[\"r\", 5, 22 | null]`. The server processing the transaction should replace that value with 23 | whatever the observed value is for that key: `[\"r\", 5, [1, 2]]`. 24 | 25 | An *append* adds an element to the end of the key's current value. For 26 | instance, `[\"append\", 5, 3]` means \"add 3 to the end of the list for key 27 | 5.\" If key 5 were currently `[1, 2]`, the resulting value would become `[1, 28 | 2, 3]`. Appends have values provided by the client, and are returned 29 | unchanged. 30 | 31 | For example, assume the current state of the database is `{1 [8]}`, and you 32 | receive a request body like: 33 | 34 | ```json 35 | {\"type\": \"txn\", 36 | \"msg_id\": 1, 37 | \"txn\": [[\"r\", 1, null], [\"append\", 1, 6], [\"append\", 2, 9]]} 38 | ``` 39 | 40 | You might return a response like: 41 | 42 | ```json 43 | {\"type\": \"txn_ok\", 44 | \"in_reply_to\": 1, 45 | \"txn\": [[\"r\", 1, [8]], [\"append\", 1, 6], [\"append\", 2, 9]]} 46 | ``` 47 | 48 | First you read the current value of key 1, returning the list [8]. Then you 49 | append 6 to key 1. Then you append 9 to key 2, implicitly creating it. The 50 | resulting state of the database would be `{1 [8, 6], 2 [9]}`. 51 | 52 | Appends in this workload are always integers, and are unique per key. Key 53 | `x` will only ever see at most one append of `0`, at most one append of `1`, 54 | and so on. 55 | 56 | Unlike lin-kv, nonexistent keys should be returned as `null`. Lists are 57 | implicitly created on first append. 58 | 59 | This workload can check many kinds of consistency models. See the 60 | `--consistency-models` CLI option for details." 61 | (:refer-clojure :exclude [read]) 62 | (:require [elle.core :as elle] 63 | [maelstrom [client :as c] 64 | [net :as net]] 65 | [jepsen [client :as client] 66 | [generator :as gen] 67 | [independent :as independent]] 68 | [jepsen.tests.cycle.append :as append] 69 | [schema.core :as s])) 70 | 71 | (def Key s/Any) 72 | (def Element s/Any) 73 | 74 | (def ReadReq [(s/one (s/eq "r") "f") (s/one Key "k") (s/one (s/eq nil) "v")]) 75 | (def ReadRes [(s/one (s/eq "r") "f") (s/one Key "k") (s/one [Element] "v")]) 76 | (def Append [(s/one (s/eq "append") "f") (s/one Key "k") (s/one Element "v")]) 77 | 78 | (c/defrpc txn! 79 | "Requests that the node execute a single transaction. Servers respond with a 80 | `txn_ok` message, and a completed version of the requested transaction; e.g. 81 | with read values filled in. Keys and list elements may be of any type." 82 | {:type (s/eq "txn") 83 | :txn [(s/either ReadReq Append)]} 84 | {:type (s/eq "txn_ok") 85 | :txn [(s/either ReadRes Append)]}) 86 | 87 | (defn kw->str 88 | "We use keywords for our :f's. Converts keywords to strings in a txn." 89 | [txn] 90 | (mapv (fn [[f k v]] 91 | [(name f) k v]) 92 | txn)) 93 | 94 | (defn str->kw 95 | "We use keywords for our :f's. Converts strings to keywords in a txn." 96 | [txn] 97 | (mapv (fn [[f k v]] 98 | [(keyword f) k v]) 99 | txn)) 100 | 101 | (defn client 102 | "Construct a linearizable key-value client for the given network" 103 | ([net] 104 | (client net nil nil)) 105 | ([net conn node] 106 | (reify client/Client 107 | (open! [this test node] 108 | (client net (c/open! net) node)) 109 | 110 | (setup! [this test]) 111 | 112 | (invoke! [_ test op] 113 | (c/with-errors op #{} 114 | (->> (:value op) 115 | kw->str 116 | (array-map :txn) 117 | (txn! conn node) 118 | :txn 119 | str->kw 120 | (assoc op :type :ok, :value)))) 121 | 122 | (teardown! [_ test]) 123 | 124 | (close! [_ test] 125 | (c/close! conn)) 126 | 127 | client/Reusable 128 | (reusable? [this test] 129 | true)))) 130 | 131 | (defn workload 132 | "Constructs a workload for linearizable registers, given option from the CLI 133 | test constructor. Options are: 134 | 135 | :net A Maelstrom network 136 | :key-count Number of keys to work on at any single time 137 | :min-txn-length Minimum number of ops per transaction 138 | :max-txn-length Maximum number of ops per transaction 139 | :max-writes-per-key How many elements to (try) appending to a single key. 140 | :consistency-models What kinds of consistency models to check for." 141 | [opts] 142 | (-> (append/test opts) 143 | (assoc :client (client (:net opts))))) 144 | -------------------------------------------------------------------------------- /src/maelstrom/workload/unique_ids.clj: -------------------------------------------------------------------------------- 1 | (ns maelstrom.workload.unique-ids 2 | "A simple workload for ID generation systems. Clients ask servers to generate 3 | an ID, and the server should respond with an ID. The test verifies that those 4 | IDs are globally unique. 5 | 6 | Your node will receive a request body like: 7 | 8 | ```json 9 | {\"type\": \"generate\", 10 | \"msg_id\": 2} 11 | ``` 12 | 13 | And should respond with something like: 14 | 15 | ```json 16 | {\"type\": \"generate_ok\", 17 | \"in_reply_to\": 2, 18 | \"id\": 123} 19 | ``` 20 | 21 | IDs may be of any type--strings, booleans, integers, floats, compound JSON 22 | values, etc." 23 | (:require [maelstrom [client :as c] 24 | [net :as net]] 25 | [jepsen [checker :as checker] 26 | [client :as client] 27 | [generator :as gen] 28 | [tests :as tests]] 29 | [schema.core :as s])) 30 | 31 | (c/defrpc generate! 32 | "Asks a node to generate a new ID. Servers respond with a generate_ok message 33 | containing an `id` field, which should be a globally unique value. IDs may be 34 | of any type." 35 | {:type (s/eq "generate")} 36 | {:type (s/eq "generate_ok") 37 | :id s/Any}) 38 | 39 | (defn client 40 | "Constructs a client for ID generation." 41 | ([net] 42 | (client net nil nil)) 43 | ([net conn node] 44 | (reify client/Client 45 | (open! [_ test node] 46 | (client net (c/open! net) node)) 47 | 48 | (setup! [_ test]) 49 | 50 | (invoke! [_ test op] 51 | (c/with-errors op #{} 52 | (assoc op :type :ok, :value (:id (generate! conn node {}))))) 53 | 54 | (teardown! [_ test]) 55 | 56 | (close! [_ test] 57 | (c/close! conn)) 58 | 59 | client/Reusable 60 | (reusable? [this test] 61 | true)))) 62 | 63 | (defn workload 64 | "Constructs a workload for unique ID generation, given options from the CLI 65 | test constructor. Options are: 66 | 67 | :net A Maelstrom network" 68 | [opts] 69 | (assoc tests/noop-test 70 | :client (client (:net opts)) 71 | :generator (gen/repeat {:f :generate}) 72 | :checker (checker/unique-ids))) 73 | -------------------------------------------------------------------------------- /test/maelstrom/core_test.clj: -------------------------------------------------------------------------------- 1 | (ns maelstrom.core-test 2 | (:require [clojure.test :refer :all] 3 | [maelstrom.core :refer :all])) 4 | 5 | -------------------------------------------------------------------------------- /test/maelstrom/service_test.clj: -------------------------------------------------------------------------------- 1 | (ns maelstrom.service-test 2 | (:require [clojure [pprint :refer [pprint]] 3 | [test :refer :all]] 4 | [maelstrom [service :as s]])) 5 | 6 | (deftest seq-kv-test 7 | (let [buf 16 8 | write-count (/ buf 2) 9 | ; Prepares a mutable kv by writing 1, 2, 3, ..., write-count - 1 10 | prep (fn prep [] 11 | (let [kv (s/sequential buf (s/persistent-kv))] 12 | ; Set x to 1, then 2, leaving some states unfilled 13 | (dotimes [i write-count] 14 | (let [r (s/handle! kv {:src "c0" :body {:type "write" 15 | :key :x 16 | :value i}})] 17 | (is (= "write_ok" (:type r))))) 18 | kv))] 19 | (testing "fresh client reads return old state" 20 | (let [kv (prep) 21 | reads (->> (range 64) 22 | (map (fn [i] 23 | (-> (s/handle! kv {:src (str "old-read-" i) 24 | :body {:type "read", :key :x}}) 25 | :value))) 26 | set)] 27 | (is (< 1 (count reads))))) 28 | 29 | (testing "ensuring recent read by writing something unique" 30 | (dotimes [i 8] 31 | (let [kv (prep) 32 | client (str "ensure-" i)] 33 | (s/handle! kv {:src client 34 | :body {:type "write" :key :ensure :value i}}) 35 | (let [r (s/handle! kv {:src client 36 | :body {:type "read", :key :x}})] 37 | (is (= (dec write-count) (:value r))))))) 38 | 39 | (testing "ensuring recent reads by reading a ton" 40 | (dotimes [i 8] 41 | (let [kv (prep) 42 | target (dec write-count) 43 | tries (* buf 10)] 44 | (loop [trial 1] 45 | (if (= trial tries) 46 | (is false) ; Never converged 47 | (let [r (s/handle! kv {:src "r", :body {:type "read", :key :x}})] 48 | ;(prn (:value r) target) 49 | (if (= target (:value r)) 50 | (do ;(println "Converged after" trial "attempts") 51 | (is true)) 52 | (recur (inc trial))))))))) 53 | )) 54 | -------------------------------------------------------------------------------- /test/maelstrom/workload/pn_counter_test.clj: -------------------------------------------------------------------------------- 1 | (ns maelstrom.workload.pn-counter-test 2 | (:require [clojure [pprint :refer [pprint]] 3 | [test :refer :all]] 4 | [jepsen.checker :as checker] 5 | [maelstrom.workload.pn-counter :refer :all])) 6 | 7 | (let [check (fn [history] (checker/check (checker) {} history {}))] 8 | (deftest checker-test 9 | (testing "empty" 10 | (is (= {:valid? true 11 | :errors nil 12 | :final-reads [] 13 | :acceptable [[0 0]]} 14 | (check [])))) 15 | 16 | (testing "definite" 17 | (is (= {:valid? false 18 | :errors [{:type :ok, :f :read, :final? true, :value 4}] 19 | :final-reads [5 4] 20 | :acceptable [[5 5]]} 21 | (check [{:type :ok, :f :add, :value 2} 22 | {:type :ok, :f :add, :value 3} 23 | {:type :ok, :f :read, :final? true, :value 5} 24 | {:type :ok, :f :read, :final? true, :value 4}])))) 25 | 26 | (testing "indefinite" 27 | (is (= {:valid? false 28 | :errors [{:type :ok, :f :read, :final? true, :value 11}] 29 | :final-reads [11 15] 30 | :acceptable [[8 10] [13 15]]} 31 | (check [{:type :ok, :f :add, :value 10} 32 | {:type :info, :f :add, :value 5} 33 | {:type :info, :f :add, :value -1} 34 | {:type :info, :f :add, :value -1} 35 | {:type :ok, :f :read, :final? true, :value 11} 36 | {:type :ok, :f :read, :final? true, :value 15}])))))) 37 | 38 | --------------------------------------------------------------------------------