├── vessel.dhall ├── example ├── vessel.dhall ├── dfx.json ├── package-set.dhall ├── TextLogger.did └── TextLogger.mo ├── test ├── Logger.mo └── Makefile ├── shell.nix ├── .github └── workflows │ └── ci.yml ├── package-set.dhall ├── LICENSE ├── README.md └── src └── Logger.mo /vessel.dhall: -------------------------------------------------------------------------------- 1 | { 2 | dependencies = [ "base", "matchers" ], 3 | compiler = None Text 4 | } 5 | -------------------------------------------------------------------------------- /example/vessel.dhall: -------------------------------------------------------------------------------- 1 | { 2 | dependencies = [ "base", "matchers", "ic-logger" ], 3 | compiler = None Text 4 | } 5 | -------------------------------------------------------------------------------- /test/Logger.mo: -------------------------------------------------------------------------------- 1 | import Logger "../src/Logger"; 2 | 3 | import Suite "mo:matchers/Suite"; 4 | 5 | Suite.run(Logger.test()) 6 | -------------------------------------------------------------------------------- /example/dfx.json: -------------------------------------------------------------------------------- 1 | { 2 | "canisters": { 3 | "logger": { 4 | "main": "TextLogger.mo", 5 | "type": "motoko" 6 | } 7 | }, 8 | "defaults": { 9 | "build": { 10 | "args": "", 11 | "packtool": "vessel sources" 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /shell.nix: -------------------------------------------------------------------------------- 1 | { pkgs ? import {} }: 2 | let 3 | ic-utils = import (builtins.fetchGit { 4 | url = "https://github.com/ninegua/ic-utils"; 5 | rev = "6e9b645e667fb59f51ad8cfa40e2fd7fdd7e52d0"; 6 | ref = "refs/heads/main"; 7 | }) { inherit pkgs; }; 8 | in pkgs.mkShell { nativeBuildInputs = [ ic-utils pkgs.wasmtime ]; } 9 | -------------------------------------------------------------------------------- /test/Makefile: -------------------------------------------------------------------------------- 1 | TESTS=Logger 2 | TARGETS=$(TESTS:%=run/%) 3 | OBJS=$(TESTS:%=dist/%.wasm) 4 | 5 | all: $(TARGETS) 6 | 7 | run/%: dist/%.wasm 8 | wasmtime $< 9 | 10 | dist: 11 | @mkdir -p $@ 12 | 13 | dist/%.wasm: %.mo ../src/%.mo | dist 14 | moc $$(vessel sources) -wasi-system-api -o $@ $< 15 | 16 | clean: 17 | rm -rf dist 18 | 19 | .PHONY: all clean 20 | .PRECIOUS: ${OBJS} 21 | -------------------------------------------------------------------------------- /example/package-set.dhall: -------------------------------------------------------------------------------- 1 | let upstream = https://github.com/dfinity/vessel-package-set/releases/download/mo-0.6.18-20220107/package-set.dhall 2 | let additions = [ 3 | { name = "ic-logger" 4 | , repo = "https://github.com/ninegua/ic-logger" 5 | , version = "95e43be3fcc285121b5bb1357bfd617efc2b2234" 6 | , dependencies = [ "base" ] 7 | } 8 | ] 9 | in upstream # additions 10 | -------------------------------------------------------------------------------- /example/TextLogger.did: -------------------------------------------------------------------------------- 1 | type View = 2 | record { 3 | messages: vec text; 4 | start_index: nat; 5 | }; 6 | type Stats = 7 | record { 8 | bucket_sizes: vec nat; 9 | start_index: nat; 10 | }; 11 | service : { 12 | allow: (vec principal) -> () oneway; 13 | append: (vec text) -> () oneway; 14 | pop_buckets: (nat) -> () oneway; 15 | stats: () -> (Stats) query; 16 | view: (nat, nat) -> (View) query; 17 | } 18 | 19 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: "ci" 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | ci: 7 | name: "ci" 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v2.3.4 11 | - uses: cachix/install-nix-action@v13 12 | with: 13 | nix_path: nixpkgs=channel:nixos-21.05 14 | - uses: cachix/cachix-action@v10 15 | with: 16 | name: ninegua 17 | authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' 18 | - run: nix-shell --run 'make -C test' 19 | -------------------------------------------------------------------------------- /package-set.dhall: -------------------------------------------------------------------------------- 1 | let upstream = https://github.com/dfinity/vessel-package-set/releases/download/mo-0.6.18-20220107/package-set.dhall 2 | let Package = 3 | { name : Text, version : Text, repo : Text, dependencies : List Text } 4 | 5 | let 6 | -- This is where you can add your own packages to the package-set 7 | additions = 8 | [] : List Package 9 | 10 | let 11 | {- This is where you can override existing packages in the package-set 12 | 13 | For example, if you wanted to use version `v2.0.0` of the foo library: 14 | let overrides = [ 15 | { name = "foo" 16 | , version = "v2.0.0" 17 | , repo = "https://github.com/bar/foo" 18 | , dependencies = [] : List Text 19 | } 20 | ] 21 | -} 22 | overrides = 23 | [] : List Package 24 | 25 | in upstream # additions # overrides 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Paul Liu 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /example/TextLogger.mo: -------------------------------------------------------------------------------- 1 | // Persistent logger keeping track of what is going on. 2 | 3 | import Array "mo:base/Array"; 4 | import Buffer "mo:base/Buffer"; 5 | import Deque "mo:base/Deque"; 6 | import List "mo:base/List"; 7 | import Nat "mo:base/Nat"; 8 | import Option "mo:base/Option"; 9 | 10 | import Logger "mo:ic-logger/Logger"; 11 | 12 | shared(msg) actor class TextLogger() { 13 | let OWNER = msg.caller; 14 | 15 | stable var state : Logger.State = Logger.new(0, null); 16 | let logger = Logger.Logger(state); 17 | 18 | // Principals that are allowed to log messages. 19 | stable var allowed : [Principal] = [OWNER]; 20 | 21 | // Set allowed principals. 22 | public shared (msg) func allow(ids: [Principal]) { 23 | assert(msg.caller == OWNER); 24 | allowed := ids; 25 | }; 26 | 27 | // Add a set of messages to the log. 28 | public shared (msg) func append(msgs: [Text]) { 29 | assert(Option.isSome(Array.find(allowed, func (id: Principal) : Bool { msg.caller == id }))); 30 | logger.append(msgs); 31 | }; 32 | 33 | // Return log stats, where: 34 | // start_index is the first index of log message. 35 | // bucket_sizes is the size of all buckets, from oldest to newest. 36 | public query func stats() : async Logger.Stats { 37 | logger.stats() 38 | }; 39 | 40 | // Return the messages between from and to indice (inclusive). 41 | public shared query (msg) func view(from: Nat, to: Nat) : async Logger.View { 42 | assert(msg.caller == OWNER); 43 | logger.view(from, to) 44 | }; 45 | 46 | // Drop past buckets (oldest first). 47 | public shared (msg) func pop_buckets(num: Nat) { 48 | assert(msg.caller == OWNER); 49 | logger.pop_buckets(num) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # IC Logger 2 | 3 | This [motoko] library provides a module to help create an append-only logger actor. 4 | 5 | ## Usage 6 | 7 | You can use this library with the [vessel] package manager. 8 | A sample usage of creating a logger actor (or canister) from the [Logger](./src/Logger.mo) module can be found in the [example](./example/) sub-directory. 9 | You'll need both [dfx] and [vessel] in PATH before trying it out: 10 | 11 | ``` 12 | cd example 13 | dfx deploy 14 | ``` 15 | 16 | It creates a text-based logger and gives the controller a few privileged methods to use it. 17 | [Its actor interface](./example/TextLogger.did) is given in [candid]: 18 | 19 | ``` 20 | type View = 21 | record { 22 | messages: vec text; 23 | start_index: nat; 24 | }; 25 | type Stats = 26 | record { 27 | bucket_sizes: vec nat; 28 | start_index: nat; 29 | }; 30 | service : { 31 | allow: (vec principal) -> () oneway; // Allow some canisters to call the `append` method. 32 | append: (vec text) -> () oneway; // Append a set of new log entries. 33 | pop_buckets: (nat) -> () oneway; // Remove the given number of oldest buckets. 34 | stats: () -> (Stats) query; // Get the latest logger stats. 35 | view: (nat, nat) -> (View) query; // View logs in the given index interval (inclusive). 36 | } 37 | ``` 38 | 39 | ## Functionality 40 | 41 | 1. The logger actor provides a single `append` method for other actors to call. 42 | 2. The logger will keep all past logs in the order as they were received, and every logline has an index or line number. 43 | These logs are stored in "buckets", and when a bucket is full, the next bucket is created. 44 | Buckets are also numbered, and the capacity of a bucket is configurable when initializing the `Logger` class. 45 | 3. The controller can perform some adminstrative duties, such as changing which canisters are allowed to call `append`, or remove old buckets to save some space. 46 | 4. When an old bucket is removed, it does not change the index of log lines. This means querying logs using `view(..)` should given consistent results (unless they are removed). 47 | 48 | ## Tips 49 | 50 | Something I find very useful when doing logging is to keep track of "sessions". 51 | Due to the nature of async programming, calls may be intertwined, so are the log messages they produce. 52 | This makes it hard to figure out what is going on in different threads (or "sessions") of execution. 53 | 54 | Here is a helper function I use to solve this problem: 55 | 56 | ``` 57 | func logger(name: Text) : Text -> async () { 58 | let prefix = "[" # Int.toText(Time.now()) # "/"; 59 | func(s: Text) : async () { 60 | Logger.append([prefix # Int.toText(Time.now() / 1_000_000_000) # "] " # name # ": " # s]) 61 | } 62 | }; 63 | ``` 64 | 65 | Calling `logger("some name")` will return a function that can be used to do the actual logging. 66 | The interesting bit is that we use the timestamp at the creation of this logger as a prefix to help distinguish "sessions" that span across `await` statements. 67 | 68 | Here are some example log entries from [tipjar]: 69 | 70 | ``` 71 | "[1642195271679220110/1642195271] heartbeat: BeforeCheck {canister = rkp4c-7iaaa-aaaaa-aaaca-cai}" 72 | "[1642195271679220110/1642195271] heartbeat: AfterCheck {cycle = 90_000_000_000_000}" 73 | "[1642195271062468878/1642195272] heartbeat: AfterCheck {cycle = 100_000_000_000_000}" 74 | "[1642198872792672023/1642198872] heartbeat: BeforeCheck {canister = rno2w-sqaaa-aaaaa-aaacq-cai}" 75 | "[1642198873407808427/1642198873] heartbeat: BeforeCheck {canister = rkp4c-7iaaa-aaaaa-aaaca-cai}" 76 | "[1642198873407808427/1642198873] heartbeat: AfterCheck {cycle = 90_000_000_000_000}" 77 | "[1642198872792672023/1642198874] heartbeat: AfterCheck {cycle = 100_000_000_000_000}" 78 | ``` 79 | 80 | It has a `heartbeat` method that logs `BeforeCheck` and `AfterCheck`. 81 | These entries are ordered by when the Logger receives the `append` call. 82 | We can see that `1642198872792672023` has two entries that do not appear next to each other. 83 | And yet we can still tell they are from the same "session" because of the common prefix. 84 | 85 | ## Development 86 | 87 | The code is documented inline and unit tests are provided. 88 | If you have installed a [nix] environment, you can run the tests like this: 89 | 90 | ``` 91 | nix-shell 92 | cd test 93 | make 94 | ``` 95 | 96 | [motoko]: https://github.com/dfinity/motoko 97 | [vessel]: https://github.com/dfinity/vessel 98 | [candid]: https://github.com/dfinity/candid 99 | [dfx]: https://github.com/dfinity/sdk 100 | [nix]: https://github.com/NixOS/nix 101 | [tipjar]: https://github.com/ninegua/tipjar 102 | -------------------------------------------------------------------------------- /src/Logger.mo: -------------------------------------------------------------------------------- 1 | import Array "mo:base/Array"; 2 | import Buffer "mo:base/Buffer"; 3 | import Deque "mo:base/Deque"; 4 | import List "mo:base/List"; 5 | import Nat "mo:base/Nat"; 6 | import Option "mo:base/Option"; 7 | 8 | import Matchers "mo:matchers/Matchers"; 9 | import T "mo:matchers/Testable"; 10 | import Suite "mo:matchers/Suite"; 11 | import Text "mo:base/Text"; 12 | import Iter "mo:base/Iter"; 13 | 14 | module { 15 | 16 | public type Buckets = Deque.Deque<[A]>; 17 | public type Bucket = List.List ; 18 | 19 | public type State = { 20 | var buckets: Buckets ; // Past buckets (left to right = newest to oldest) 21 | var num_of_buckets: Nat; // Number of past buckets 22 | var bucket: Bucket ; // Current bucket (tail to head = newest to oldest) 23 | var num_of_lines: Nat; // Number of lines in current bucket 24 | var start_index: Nat; // Start index of the first message in past buckets 25 | bucket_size: Nat; 26 | }; 27 | 28 | public type Stats = { 29 | start_index: Nat; 30 | bucket_sizes: [Nat]; 31 | }; 32 | 33 | public type View = { 34 | start_index: Nat; 35 | messages: [A]; 36 | }; 37 | 38 | let BUCKET_SIZE = 5000; // Default bucket size 39 | 40 | // Initialize an empty logger state with the given start_index and bucket_size. 41 | public func new (start_index: Nat, bucket_size: ?Nat) : State { 42 | { 43 | var buckets : Buckets = Deque.empty(); 44 | var num_of_buckets = 0; 45 | var bucket : Bucket = List.nil(); 46 | var num_of_lines = 0; 47 | var start_index = start_index; 48 | bucket_size = Option.get(bucket_size, BUCKET_SIZE); 49 | } 50 | }; 51 | 52 | // Convert a list to array (ordered from head to tail). 53 | public func to_array(list: List.List, n: Nat) : [T] { 54 | let buf = Buffer.Buffer(n); 55 | var l = list; 56 | label LOOP loop { 57 | switch (List.pop(l)) { 58 | case (null, _) { break LOOP; }; 59 | case (?v, l_) { buf.add(v); l := l_; } 60 | } 61 | }; 62 | Array.tabulate(n, func(i: Nat): T { buf.get(n - i - 1) }) 63 | }; 64 | 65 | public class Logger (s : State ) { 66 | 67 | // Move bucket into buckets 68 | public func roll_over() { 69 | s.buckets := Deque.pushBack(s.buckets, to_array(s.bucket, s.num_of_lines)); 70 | s.num_of_buckets := s.num_of_buckets + 1; 71 | s.bucket := List.nil(); 72 | s.num_of_lines := 0; 73 | }; 74 | 75 | // Add a set of messages to the log. 76 | public func append(msgs: [A]) { 77 | for (msg in msgs.vals()) { 78 | s.bucket := List.push(msg, s.bucket); 79 | s.num_of_lines := s.num_of_lines + 1; 80 | if (s.num_of_lines >= s.bucket_size) { 81 | roll_over() 82 | } 83 | } 84 | }; 85 | 86 | // Return log stats, where: 87 | // start_index is the first index of log message. 88 | // bucket_sizes is the size of all buckets, from oldest to newest. 89 | public func stats() : Stats { 90 | var bucket_sizes = Array.init(s.num_of_buckets + 1, 0); 91 | var i = s.num_of_buckets; 92 | bucket_sizes[i] := s.num_of_lines; 93 | var bs = s.buckets; 94 | label LOOP loop { 95 | switch (Deque.popBack(bs)) { 96 | case null { break LOOP; }; 97 | case (?(bs_, b)) { 98 | i := i - 1; 99 | bucket_sizes[i] := b.size(); 100 | bs := bs_; 101 | } 102 | } 103 | }; 104 | { start_index = s.start_index; bucket_sizes = Array.freeze(bucket_sizes) } 105 | }; 106 | 107 | // Return the messages between from and to indice (inclusive). 108 | public func view(from: Nat, to: Nat) : View { 109 | assert(to >= from); 110 | let buf = Buffer.Buffer (to - from + 1); 111 | var i = s.start_index; 112 | var b = s.buckets; 113 | label LOOP loop { 114 | switch (Deque.popFront(b)) { 115 | case null { break LOOP; }; 116 | case (?(lines, d)) { 117 | let n = lines.size(); 118 | // is there intersection between [i, i + n] and [from, to] 119 | if (i > to) { break LOOP; }; 120 | if (i + n > from) { 121 | var k = if (i < from) { Nat.sub(from, i) } else { 0 }; 122 | let m = if (i + n > to) { Nat.sub(to + 1, i) } else { n }; 123 | while (k < m) { 124 | buf.add(lines[k]); 125 | k := k + 1; 126 | } 127 | }; 128 | i := i + n; 129 | b := d; 130 | } 131 | } 132 | }; 133 | if (i + s.num_of_lines > from and i <= to) { 134 | let arr : [A] = to_array(s.bucket, s.num_of_lines); 135 | var k = if (i < from) { Nat.sub(from, i) } else { 0 }; 136 | let m = if (i + s.num_of_lines > to) { Nat.sub(to + 1, i) } else { s.num_of_lines }; 137 | while (k < m) { 138 | buf.add(arr[k]); 139 | k := k + 1; 140 | } 141 | }; 142 | { 143 | start_index = if (s.start_index > from) { s.start_index } else { from }; 144 | messages = buf.toArray(); 145 | } 146 | }; 147 | 148 | // Drop past buckets (oldest first). 149 | public func pop_buckets(num: Nat) { 150 | var i = 0; 151 | label LOOP while (i < num) { 152 | switch (Deque.popFront(s.buckets)) { 153 | case null { break LOOP }; 154 | case (?(b, bs)) { 155 | s.num_of_buckets := s.num_of_buckets - 1; 156 | s.buckets := bs; 157 | s.start_index := s.start_index + b.size(); 158 | }; 159 | }; 160 | i := i + 1; 161 | } 162 | } 163 | }; 164 | 165 | type Suite = Suite.Suite; 166 | public func test() : Suite { 167 | let to_array_test = 168 | Suite.suite("to_array", [ 169 | Suite.test("empty", 170 | to_array(List.nil(), 0), 171 | Matchers.equals(T.array(T.natTestable, []))), 172 | Suite.test("singleton", 173 | to_array(List.push(1, List.nil()), 1), 174 | Matchers.equals(T.array(T.natTestable, [1]))), 175 | Suite.test("multiple", 176 | to_array(List.push(2, List.push(1, List.nil())), 2), 177 | Matchers.equals(T.array(T.natTestable, [1, 2]))), 178 | Suite.test("truncated to size", 179 | to_array(List.push(3, List.push(2, List.push(1, List.nil()))), 2), 180 | Matchers.equals(T.array(T.natTestable, [2, 3]))), 181 | ]); 182 | 183 | let N = 10; 184 | let S = 3; 185 | let logger = Logger(new(0, ?S)); 186 | let append_test = Suite.suite("append", 187 | Array.tabulate(N, func(n: Nat): Suite { 188 | logger.append([n]); 189 | Suite.test(Text.concat("append/", Nat.toText(n)), 190 | logger.view(0, n).messages, 191 | Matchers.equals(T.array(T.natTestable, 192 | Array.tabulate(n + 1, func(x: Nat): Nat { x })))) 193 | })); 194 | 195 | let view_tests = Buffer.Buffer(0); 196 | for (i in Iter.range(0, N - 2)) { 197 | for (j in Iter.range(i, N - 1)) { 198 | view_tests.add( 199 | Suite.test(Text.join("/", Iter.fromArray(["view", Nat.toText(i), Nat.toText(j)])), 200 | logger.view(i, j).messages, 201 | Matchers.equals(T.array(T.natTestable, 202 | Array.tabulate(Nat.sub(j+1, i), func(x: Nat): Nat { x + i }))))); 203 | }; 204 | }; 205 | let view_test = Suite.suite("view", view_tests.toArray()); 206 | 207 | let view_tests_after_pop = Buffer.Buffer(0); 208 | for (k in Iter.range(1,2)) { 209 | logger.pop_buckets(1); 210 | for (i in Iter.range(0, N - 2)) { 211 | for (j in Iter.range(i, N - 1)) { 212 | view_tests.add( 213 | Suite.test(Text.join("/", Iter.fromArray(["view", Nat.toText(i), Nat.toText(j)])), 214 | logger.view(i, j).messages, 215 | Matchers.equals(T.array(T.natTestable, 216 | if (j + 1 > k * S) { 217 | Array.tabulate(Nat.sub(j+1, Nat.max(i, S)), func(x: Nat): Nat { x + i }) 218 | } else { [] } 219 | )))) 220 | } 221 | } 222 | }; 223 | let view_test_after_pop = Suite.suite("view", view_tests_after_pop.toArray()); 224 | 225 | Suite.suite("Test Logger", [ to_array_test, append_test, view_test, view_test_after_pop ]) 226 | } 227 | } 228 | --------------------------------------------------------------------------------