├── .java-version ├── .envrc ├── bin └── runl ├── src ├── app │ ├── utils.clj │ ├── xtdb_contrib.clj │ └── nodes.cljc ├── user.cljs ├── logback.xml └── user.clj ├── shadow-cljs.edn ├── resources └── public │ ├── index.html │ └── nodes.css ├── .gitignore ├── deps.edn └── README.md /.java-version: -------------------------------------------------------------------------------- 1 | 11.0 -------------------------------------------------------------------------------- /.envrc: -------------------------------------------------------------------------------- 1 | export XTDB_ENABLE_BYTEUTILS_SHA1=true 2 | -------------------------------------------------------------------------------- /bin/runl: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | XTDB_ENABLE_BYTEUTILS_SHA1=true clj -A:dev -X user/main 3 | -------------------------------------------------------------------------------- /src/app/utils.clj: -------------------------------------------------------------------------------- 1 | (ns app.utils) 2 | 3 | (defn created-at [] 4 | (.format (java.time.LocalDateTime/now) 5 | (java.time.format.DateTimeFormatter/ofPattern 6 | "yyyy-MM-dd HH:mm:ss"))) 7 | -------------------------------------------------------------------------------- /shadow-cljs.edn: -------------------------------------------------------------------------------- 1 | {:deps {:aliases [:dev]} 2 | :nrepl {:port 9001} 3 | :builds 4 | {:dev 5 | {:target :browser 6 | :devtools {:watch-dir "resources/public" ; live reload CSS 7 | :hud #{:errors :progress} 8 | :ignore-warnings true} ; warnings don't prevent hot-reload 9 | :output-dir "resources/public/js" 10 | :asset-path "/js" 11 | :modules {:main {:entries [user] 12 | :init-fn user/start!}}}}} 13 | -------------------------------------------------------------------------------- /resources/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | subspace 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | clj/**/pom.xml 2 | clj/**/pom.xml.asc 3 | *.jar 4 | *.class 5 | /lib/ 6 | /classes/ 7 | /target/ 8 | /checkouts/ 9 | .lein-deps-sum 10 | .lein-repl-history 11 | .lein-plugins/ 12 | .lein-failures 13 | .nrepl-port 14 | .cpcache/ 15 | .lsp/ 16 | clj/data/ 17 | clj/target/ 18 | java/data/ 19 | java/target/ 20 | java/.classpath 21 | java/.project 22 | .settings/ 23 | .vscode/ 24 | *.~undo-tree~ 25 | .DS_Store 26 | /data/ 27 | /resources/public/js/ 28 | /.shadow-cljs/ 29 | .calva/ 30 | .clj-kondo/ 31 | -------------------------------------------------------------------------------- /src/user.cljs: -------------------------------------------------------------------------------- 1 | (ns ^:dev/always user ; Electric currently needs to rebuild everything when any file changes. Will fix 2 | (:require 3 | app.nodes 4 | hyperfiddle.electric 5 | hyperfiddle.electric-dom2)) 6 | 7 | (def electric-main 8 | (hyperfiddle.electric/boot ; Electric macroexpansion - Clojure to signals compiler 9 | (binding [hyperfiddle.electric-dom2/node js/document.body] 10 | (app.nodes/Nodes.)))) 11 | 12 | (defonce reactor nil) 13 | 14 | (defn ^:dev/after-load ^:export start! [] 15 | (assert (nil? reactor) "reactor already running") 16 | (set! reactor (electric-main 17 | #(js/console.log "Reactor success:" %) 18 | #(js/console.error "Reactor failure:" %)))) 19 | 20 | (defn ^:dev/before-load stop! [] 21 | (when reactor (reactor)) ; teardown 22 | (set! reactor nil)) -------------------------------------------------------------------------------- /src/app/xtdb_contrib.clj: -------------------------------------------------------------------------------- 1 | (ns app.xtdb-contrib 2 | (:require [missionary.core :as m] 3 | [xtdb.api :as xt])) 4 | 5 | (defn latest-db> 6 | "return flow of latest XTDB tx, but only works for XTDB in-process mode. see 7 | https://clojurians.slack.com/archives/CG3AM2F7V/p1677432108277939?thread_ts=1677430221.688989&cid=CG3AM2F7V" 8 | [!xtdb] 9 | (->> (m/observe (fn [!] 10 | (let [listener (xt/listen !xtdb {::xt/event-type ::xt/indexed-tx :with-tx-ops? true} !)] 11 | #(.close listener)))) 12 | (m/reductions {} (xt/latest-completed-tx !xtdb)) ; initial value is the latest known tx, possibly nil 13 | (m/relieve {}) 14 | (m/latest (fn [{:keys [:xt/tx-time] :as ?tx}] 15 | (if tx-time (xt/db !xtdb {::xt/tx-time tx-time}) 16 | (xt/db !xtdb)))))) 17 | -------------------------------------------------------------------------------- /src/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | %highlight(%-5level) %logger: %msg%n 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /resources/public/nodes.css: -------------------------------------------------------------------------------- 1 | @import url('https://fonts.googleapis.com/css2?family=IBM+Plex+Sans:wght@100;200;300;400;500;600;700&display=swap'); 2 | 3 | body { 4 | font-family: 'IBM Plex Sans', sans-serif; 5 | font-weight: 400; 6 | } 7 | 8 | input[type="text"] { 9 | padding: 0.5rem; 10 | margin-bottom: 1rem; 11 | min-width: 30rem; 12 | border-radius: 0.25rem; 13 | border: 1px gray solid; 14 | } 15 | 16 | input[type="checkbox"] { 17 | width: 1rem; 18 | height: 1rem; 19 | } 20 | 21 | input[type="checkbox"]:checked+label { 22 | text-decoration: line-through; 23 | color: gray; 24 | } 25 | 26 | label { 27 | padding-left: 0.5rem; 28 | } 29 | 30 | label, 31 | input[type="checkbox"] { 32 | cursor: pointer; 33 | transition: 0.4s ease all; 34 | } 35 | 36 | label::first-letter { 37 | text-transform: capitalize; 38 | } 39 | 40 | .nodes { 41 | display: grid; 42 | grid-template-rows: auto fit-content auto; 43 | max-width: 30rem; 44 | } 45 | -------------------------------------------------------------------------------- /deps.edn: -------------------------------------------------------------------------------- 1 | ;; Start a REPL with env var XTDB_ENABLE_BYTEUTILS_SHA1=true 2 | {:paths ["src" "resources"] 3 | :deps 4 | {ch.qos.logback/logback-classic {:mvn/version "1.2.11"} 5 | ;com.google.guava/guava {:mvn/version "31.1-jre"} ; force guava to latest to fix shadow issue 6 | com.hyperfiddle/electric {:mvn/version "v2-alpha-167-gbd475584"} 7 | com.hyperfiddle/rcf {:mvn/version "20220926-202227"} 8 | com.xtdb/xtdb-core {:mvn/version "1.23.0"} 9 | com.xtdb/xtdb-rocksdb {:mvn/version "1.23.0"} 10 | info.sunng/ring-jetty9-adapter 11 | {:mvn/version "0.14.3" ; (Jetty 9) is Java 8 compatible; 12 | ;:mvn/version "0.17.7" ; (Jetty 10) is NOT Java 8 compatible 13 | :exclusions [org.slf4j/slf4j-api info.sunng/ring-jetty9-adapter-http3]} ; no need 14 | missionary/missionary {:mvn/version "b.27-SNAPSHOT"} 15 | org.clojure/clojure {:mvn/version "1.11.1"} 16 | org.clojure/clojurescript {:mvn/version "1.11.60"} 17 | org.clojure/tools.logging {:mvn/version "1.2.4"} 18 | org.slf4j/slf4j-api {:mvn/version "1.7.30"}} 19 | 20 | :aliases 21 | {:dev 22 | {:extra-deps 23 | {binaryage/devtools {:mvn/version "1.0.6"} 24 | thheller/shadow-cljs {:mvn/version "2.20.1"}} 25 | :jvm-opts 26 | ["-Xss2m" ; https://github.com/hyperfiddle/photon/issues/11 27 | "-Dclojure.tools.logging.factory=clojure.tools.logging.impl/slf4j-factory" 28 | "-Dlogback.configurationFile=src/logback.xml" 29 | "-XX:-OmitStackTraceInFastThrow" ;; RCF 30 | "-XX:+UnlockDiagnosticVMOptions" 31 | "-XX:+DebugNonSafepoints"] 32 | :exec-fn user/main 33 | :exec-args {}}} 34 | :jvm-opts ["-Dclojure.tools.logging.factory=clojure.tools.logging.impl/slf4j-factory" 35 | ;; the following option is required for JDK 16 and 17: 36 | ;; https://github.com/xtdb/xtdb/issues/1462 37 | "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED"]} 38 | -------------------------------------------------------------------------------- /src/user.clj: -------------------------------------------------------------------------------- 1 | (ns user ; Must be ".clj" file, Clojure doesn't auto-load user.cljc 2 | (:require clojure.java.io 3 | [xtdb.api :as xt])) 4 | 5 | ; lazy load dev stuff - for faster REPL startup and cleaner dev classpath 6 | (def start-electric-server! (delay @(requiring-resolve 'hyperfiddle.electric-jetty-server/start-server!))) 7 | (def shadow-start! (delay @(requiring-resolve 'shadow.cljs.devtools.server/start!))) 8 | (def shadow-watch (delay @(requiring-resolve 'shadow.cljs.devtools.api/watch))) 9 | 10 | (defn start-xtdb! [] ; from XTDB’s getting started: xtdb-in-a-box 11 | (assert (= "true" (System/getenv "XTDB_ENABLE_BYTEUTILS_SHA1"))) 12 | (letfn [(kv-store [dir] {:kv-store {:xtdb/module 'xtdb.rocksdb/->kv-store 13 | :db-dir (clojure.java.io/file dir) 14 | :sync? true}})] 15 | (xt/start-node 16 | {:xtdb/tx-log (kv-store "data/dev/tx-log") 17 | :xtdb/document-store (kv-store "data/dev/doc-store") 18 | :xtdb/index-store (kv-store "data/dev/index-store")}))) 19 | 20 | (def electric-server-config 21 | {:host "0.0.0.0", :port 8080, :resources-path "public"}) 22 | 23 | (def !xtdb) 24 | (def !electric-server) 25 | 26 | ; Server-side Electric userland code is lazy loaded by the shadow build. 27 | ; WARNING: make sure your REPL and shadow-cljs are sharing the same JVM! 28 | 29 | (defn main [& args] 30 | (println "Starting XTDB...") 31 | (alter-var-root #'!xtdb (constantly (start-xtdb!))) 32 | (comment (.close !xtdb)) 33 | (println "Starting Electric compiler...") 34 | (@shadow-start!) ; serves index.html as well 35 | (@shadow-watch :dev) ; depends on shadow server 36 | (println "Starting Electric server...") 37 | (alter-var-root #'!electric-server (constantly (@start-electric-server! electric-server-config))) 38 | (comment (.stop !electric-server))) 39 | 40 | (comment 41 | (main) ; Electric Clojure(JVM) REPL entrypoint 42 | (hyperfiddle.rcf/enable!) ; turn on RCF after all transitive deps have loaded 43 | 44 | ; debug XTDB protocol reloading issues 45 | (type !xtdb) 46 | (def db (xt/db !xtdb)) 47 | (xt/q db '{:find [(pull e [:xt/id :user/name])] 48 | :in [needle] 49 | :where [[e :user/name name] 50 | [(clojure.string/includes? name needle)]]} 51 | "") 52 | 53 | (shadow.cljs.devtools.api/repl :dev) ; shadow server hosts the cljs repl 54 | ; connect a second REPL instance to it 55 | ; (DO NOT REUSE JVM REPL it will fail weirdly) 56 | (type 1)) 57 | -------------------------------------------------------------------------------- /src/app/nodes.cljc: -------------------------------------------------------------------------------- 1 | (ns app.nodes 2 | (:import [hyperfiddle.electric Pending]) 3 | (:require #?(:clj [app.xtdb-contrib :as db]) 4 | #?(:clj [app.utils :as u]) 5 | [hyperfiddle.electric :as e] 6 | [hyperfiddle.electric-dom2 :as dom] 7 | [hyperfiddle.electric-ui4 :as ui] 8 | [xtdb.api #?(:clj :as :cljs :as-alias) xt])) 9 | 10 | (e/def !xtdb) 11 | (e/def db) ; injected database ref; Electric defs are always dynamic 12 | 13 | #?(:cljs 14 | (defn wait-for-element 15 | [element-id callback] 16 | (let [check-element (fn [interval-id-atom] 17 | (when-let [element (js/document.getElementById element-id)] 18 | (js/clearInterval @interval-id-atom) 19 | (reset! interval-id-atom nil) 20 | (callback element))) 21 | interval-id-atom (atom nil)] 22 | (reset! interval-id-atom (js/setInterval (partial check-element interval-id-atom) 50))))) 23 | 24 | #?(:clj 25 | (defn upsert-li! [!xtdb & [?node ?text]] 26 | {:pre [!xtdb]} 27 | (let [id (:xt/id ?node (random-uuid)) 28 | doc (if ?node 29 | (merge ?node {:node/text ?text}) 30 | {:xt/id id 31 | :node/text (or ?text "") 32 | :node/type :text 33 | :node/created-at (u/created-at)})] 34 | 35 | (xt/submit-tx !xtdb [[:xtdb.api/put doc]]) 36 | id))) 37 | 38 | (e/defn Node [id] 39 | (e/server 40 | (let [node (xt/entity db id)] 41 | (e/client 42 | (println "rendering" node) 43 | (dom/li (dom/props {:class "node"}) 44 | (dom/div (dom/props {:class "node-text" 45 | :id (str "node-text-" id) 46 | :contenteditable true}) 47 | (set! (.-innerText dom/node) (-> node :node/text str)) 48 | (dom/on "keydown" 49 | (e/fn [e] 50 | (println (.-key e)) 51 | (cond (= "Enter" (.-key e)) 52 | (let [v (-> e .-target .-innerText)] 53 | (.preventDefault e) 54 | (e/server 55 | (upsert-li! !xtdb node v) ; save current node 56 | (let [id (upsert-li! !xtdb nil "")] ; create new node 57 | (e/client (wait-for-element (str "node-text-" id) 58 | (fn [e] (.focus e))))))) 59 | 60 | (and (= "Backspace" (.-key e)) 61 | (= 0 (count (.. e -target -innerText)))) 62 | (e/server 63 | (e/discard 64 | (xt/submit-tx !xtdb [[::xt/delete id]])))))) 65 | (dom/on "blur" 66 | (e/fn [e] 67 | (let [text (.. e -target -innerText)] 68 | (e/server 69 | (e/discard 70 | (xt/submit-tx !xtdb [[:xtdb.api/put (merge node {:node/text text})]])))))))))))) 71 | 72 | #?(:clj 73 | (defn root-nodes [db] 74 | (->> (xt/q db '{:find [(pull ?e [*]) ?created-at] 75 | :where [[?e :node/created-at ?created-at]] 76 | :order-by [[?created-at :asc]]}) 77 | (map first) 78 | vec))) 79 | 80 | (comment 81 | (root-nodes (xt/db user/!xtdb)) 82 | (upsert-li! user/!xtdb)) 83 | 84 | (e/defn Nodes [] 85 | (e/server 86 | (binding [!xtdb user/!xtdb 87 | db (new (db/latest-db> user/!xtdb))] 88 | (e/client 89 | (dom/link (dom/props {:rel :stylesheet :href "/nodes.css"})) 90 | (try 91 | (dom/div (dom/props {:class "nodes"}) 92 | (dom/ul 93 | (e/server 94 | (e/for-by :xt/id [{:keys [xt/id]} (root-nodes db) #_(e/offload #(root-nodes db))] ; Pending is causing over-rendering, fixme 95 | (Node. id))))) 96 | (catch Pending _ 97 | (dom/props {:style {:background-color "#e1e1e1"}}))))))) 98 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # subspace.ai - PKM + REPL + AI 2 | 3 | The long-term goal of subspace is to be/have three things: 4 | 5 | * PKM (Personal Knowledge Management system) like Roam Research or Tana. 6 | * REPL-like (Read Evaluate Print Loop) capabilities. Should be able to execute individual code cells in the JVM backend and rendered in the frontend with [Electric](https://github.com/hyperfiddle/electric). Similar behaviour can be achieved with other languages via Jupyter kernels (or GraalVM Polyglot) and JavaScript. 7 | * AI (Artificial Intelligence) integrations. Should be integrated with LLMs - e.g. write GPT queries in subspace, and incorporate the response to your personal knowledge base as a new node. Intelligent search and LLM-based summaries and reasoning over the existing knowledge base (Retrieval Oriented Generation, RAG). 8 | 9 | The overall design should be open-ended, allowing for easy forking and providing custom node types / rendering functions. The goal is not to be just a storage of information, but a control panel for commonly used workflows. So that you can create convenient shortcuts and informative output views with Clojure + Electric. Since you persist which actions you took over time, you can search for past outputs and interleave these with your personal notes. Later query your knowledge base with RAG in natural language, or query it with GPT by exposing subspace knowledge base as an API to GPT. 10 | 11 | For example, additional customizations and use cases could be: 12 | 13 | * Intelligent work log for day to day coding. 14 | * Wrappers for any babashka / shell scripts you already have. 15 | * Wrapper functions to MLOps platform (or some other task manager) to trigger jobs, query stats and logs from past train runs. Build dashboards as subspace nodes from the result of such queries with Electric+HTML. 16 | * Wrappers for common Kubernetes / AWS / GCP commands. Build ad hoc UIs on top of your cluster that make sense to *you*. 17 | * Wrappers that pull the contents of arxiv documents as subspace nodes. 18 | * Spaced repetition learning of content (of nodes which you mark to be remembered). 19 | 20 | ## UI/UX 21 | 22 | There will be two types of UI elements: pages and nodes. Pages contain nodes, and nodes can nest other nodes. Both pages and nodes are referencable (meaning you can link to them and the page/node will get a backreference). 23 | 24 | Each node contains some media, and possibly subnodes. 25 | 26 | Media can be: 27 | 1. Text, numeric, Markdown 28 | 2. Image, video, audio 29 | 3. Flexible spreadsheet [tesserrae](https://github.com/lumberdev/tesserae) 30 | 4. code block, which can be executed in a jupyter kernel (runs once) 31 | 5. code block containing an e/fn (runs continuously when on the page) 32 | 33 | Executing an e/fn is the most powerful and flexible thing to do. It can pull data in from other nodes on the page or in the graph, and displays its own little UI within its boundaries. Crucially, when upstream info changes, your e/fn's output gets recomputed. Running tesserrae is also very powerful; you can think of subspace as a non-grid tesserae that can also embed tesserae. 34 | 35 | Subnodes can be organised either by indenting or tiling. 36 | 1. Indented - subnodes are nested via indentation (like sub bullet points) 37 | 2. Tiled - subnodes are split horizontally or vertically 38 | 39 | Each node is configured separately which of the three subnode organisation approaches is used. For example, a single page can have several tiles to split up the area into broad regions (using horizontal and vertical tiling) and within each tile we have nested bullet points of text (indented way of organising subnodes). 40 | 41 | ## Setup 42 | 43 | ``` 44 | brew tap homebrew/cask-versions 45 | brew install --cask temurin11 46 | brew install clojure/tools/clojure 47 | 48 | brew install jenv 49 | jenv add /Library/Java/JavaVirtualMachines/temurin-11.jdk/Contents/Home/ 50 | ``` 51 | 52 | ## Running: 53 | 54 | ``` 55 | $ XTDB_ENABLE_BYTEUTILS_SHA1=true clj -A:dev -X user/main 56 | 57 | Starting Electric compiler and server... 58 | shadow-cljs - server version: 2.20.1 running at http://localhost:9630 59 | shadow-cljs - nREPL server started on port 9001 60 | [:app] Configuring build. 61 | [:app] Compiling ... 62 | [:app] Build completed. (224 files, 0 compiled, 0 warnings, 1.93s) 63 | 64 | 👉 App server available at http://0.0.0.0:8080 65 | ``` 66 | 67 | ## Why the name? 68 | 69 | In mathematics, a *subspace* is some region in a vector space. In machine learning, we often operate with neural embedding vectors (of text, images, etc) where embeddings of similar concepts have vectors nearby in this "[latent space](https://en.wikipedia.org/wiki/Latent_space)". 70 | 71 | As a metaphor, you can imagine each node in your PKM tool to occupy one point in a very high-dimensional latent space of knowledge. Navigating to a node in the PKM is like zooming in to a region in this latent space, which contains many nodes (points) nested under the main node - so in a way, you're viewing a subspace of your knowledge base. 72 | 73 | ## Status 74 | 75 | Almost nothing started, but might scope out the UI when Electric v3 comes out. 76 | 77 | ## Credits 78 | 79 | * Adapted from [electric-xtdb-started](https://github.com/hyperfiddle/electric-xtdb-starter) 80 | * Which was adapted from [xtdb-in-a-box](https://github.com/xtdb/xtdb-in-a-box) 81 | 82 | Contributions welcome! Raise an issue/PR or reach out via Twittwer @mattiasarro. 83 | --------------------------------------------------------------------------------