}'
62 | ```
63 |
64 | If the target content is an image, include `"content_type":"image"` in the
65 | `--meta` object.
66 |
67 | If this is interesting to you, swing by this
68 | [Github issue](https://github.com/cablehead/stacks/issues/50).
69 |
--------------------------------------------------------------------------------
/examples/x-macos-pasteboard/solid-ui/deno.json:
--------------------------------------------------------------------------------
1 | {
2 | "tasks": {
3 | "dev": "deno run -A --node-modules-dir npm:vite",
4 | "build": "deno run -A --node-modules-dir npm:vite build",
5 | "preview": "deno run -A --node-modules-dir npm:vite preview",
6 | "serve": "deno run --allow-net --allow-read jsr:@std/http@1/file-server dist/"
7 | },
8 | "compilerOptions": {
9 | "lib": ["ES2020", "DOM", "DOM.Iterable"],
10 | "jsx": "react-jsx",
11 | "jsxImportSource": "solid-js"
12 | },
13 | "imports": {
14 | "@deno/vite-plugin": "npm:@deno/vite-plugin@^1.0.0",
15 | "solid-js": "npm:solid-js@^1.9.2",
16 | "solid-styled-components": "npm:solid-styled-components@^0.28.5",
17 | "vite": "npm:vite@^5.4.9",
18 | "vite-plugin-solid": "npm:vite-plugin-solid@^2.10.2"
19 | }
20 | }
21 |
--------------------------------------------------------------------------------
/examples/x-macos-pasteboard/solid-ui/handler-pb.map.nu:
--------------------------------------------------------------------------------
1 | {
2 | run: {|frame|
3 | if $frame.topic != "pb.recv" { return }
4 |
5 | let data = .cas $frame.hash | from json | get types
6 |
7 | $data | get -i "public.png" | if ($in | is-not-empty) {
8 | $in | decode base64 | .append content --meta {
9 | updates: $frame.id
10 | content_type: "image"
11 | }
12 | return
13 | }
14 |
15 | $data | get -i "public.utf8-plain-text" | if ($in | is-not-empty) {
16 | $in | decode base64 | decode | .append content --meta {updates: $frame.id}
17 | return
18 | }
19 |
20 | $frame
21 | }
22 | }
23 |
--------------------------------------------------------------------------------
/examples/x-macos-pasteboard/solid-ui/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
--------------------------------------------------------------------------------
/examples/x-macos-pasteboard/solid-ui/src/App.tsx:
--------------------------------------------------------------------------------
1 | import { Component, For } from "solid-js";
2 | import { useFrameStream } from "./store/stream";
3 | import { useStore } from "./store";
4 | import { createCAS } from "./store/cas";
5 | import Card from "./Card";
6 |
7 | const App: Component = () => {
8 | const frameSignal = useFrameStream();
9 |
10 | const fetchContent = async (hash: string) => {
11 | const response = await fetch(`/api/cas/${hash}`);
12 | if (!response.ok) {
13 | throw new Error(`Failed to fetch content for hash ${hash}`);
14 | }
15 | return await response.text();
16 | };
17 |
18 | const { index } = useStore({ dataSignal: frameSignal });
19 | const CAS = createCAS(fetchContent);
20 |
21 | return (
22 |
23 |
a solid clipboard
24 |
25 | {(frames) => }
26 |
27 |
28 | );
29 | };
30 |
31 | export default App;
32 |
--------------------------------------------------------------------------------
/examples/x-macos-pasteboard/solid-ui/src/Card.tsx:
--------------------------------------------------------------------------------
1 | import { Component, createMemo, createSignal, For, Show } from "solid-js";
2 | import { styled } from "solid-styled-components";
3 | import { Frame } from "./store/stream";
4 | import { CASStore } from "./store/cas";
5 |
6 | const CardWrapper = styled("div")`
7 | display: flex;
8 | flex-direction: column;
9 | margin-bottom: 1em;
10 | overflow: hidden;
11 | border-radius: 0.25em;
12 | `;
13 |
14 | const Content = styled("div")`
15 | flex: 1;
16 | overflow-x: auto;
17 | overflow-y: hidden;
18 | padding: 0.25em 0.5em;
19 | `;
20 |
21 | const Meta = styled("div")`
22 | font-size: 0.80em;
23 | color: var(--color-sub-fg);
24 | background-color: var(--color-sub-bg);
25 | padding: 0.5em 1em;
26 | display: flex;
27 | align-items: center;
28 | justify-content: space-between;
29 | `;
30 |
31 | type CardProps = {
32 | frames: Frame[];
33 | CAS: CASStore;
34 | };
35 |
36 | const Card: Component = (props) => {
37 | const { frames, CAS } = props;
38 | const [currentIndex, setCurrentIndex] = createSignal(0);
39 | const frame = () => frames[currentIndex()];
40 | const contentSignal = () => CAS.get(frame().hash);
41 |
42 | const renderContent = () => {
43 | const content = contentSignal()();
44 | if (!content) return null;
45 |
46 | if (frame().topic === "pb.recv") {
47 | try {
48 | const jsonContent = JSON.parse(content);
49 | return {JSON.stringify(jsonContent, null, 2)};
50 | } catch (error) {
51 | console.error("Failed to parse JSON content:", error);
52 | return {content}
;
53 | }
54 | } else if (frame().meta?.content_type === "image") {
55 | return .hash}`})
;
56 | } else {
57 | return {content};
58 | }
59 | };
60 |
61 | // Create a reactive derived signal for `source`
62 | const source = createMemo(() => {
63 | const sourceFrame = frames.find((f) => f.topic === "pb.recv");
64 | if (!sourceFrame) return null;
65 |
66 | const sourceContent = CAS.get(sourceFrame.hash)();
67 | if (!sourceContent) return null;
68 |
69 | try {
70 | const parsedContent = JSON.parse(sourceContent);
71 | return parsedContent.source;
72 | } catch (error) {
73 | console.error("Failed to parse JSON content for source:", error);
74 | return null;
75 | }
76 | });
77 |
78 | return (
79 |
80 |
81 | {frame().id}
82 |
95 |
96 | {source()}
97 |
98 |
99 | {renderContent()}
100 |
101 | );
102 | };
103 |
104 | export default Card;
105 |
--------------------------------------------------------------------------------
/examples/x-macos-pasteboard/solid-ui/src/index.css:
--------------------------------------------------------------------------------
1 | :root {
2 | font-family: ui-monospace, Menlo, Monaco, "Cascadia Mono", "Segoe UI Mono",
3 | "Roboto Mono", "Oxygen Mono", "Ubuntu Monospace", "Source Code Pro",
4 | "Fira Mono", "Droid Sans Mono", "Courier New", monospace;
5 |
6 | font-size: 1.2em;
7 | line-height: 1.5;
8 |
9 | /* Colors */
10 | --color-fg: #4e5668;
11 | --color-bg: #fff;
12 |
13 | --color-sub-fg: #7d889f;
14 | --color-sub-bg: #eceff4;
15 |
16 | --color-accent: #94bfce;
17 | }
18 |
19 | body {
20 | margin: 0 auto;
21 | padding: 1em;
22 | max-width: 620px;
23 | display: flex;
24 |
25 | color: var(--color-fg);
26 | background-color: var(--color-bg);
27 | }
28 |
29 | main {
30 | width: 100%;
31 | }
32 |
33 | pre {
34 | margin: 0;
35 | padding: 0;
36 | }
37 |
38 | iframe,
39 | img,
40 | input,
41 | select,
42 | textarea {
43 | height: auto;
44 | max-width: 100%;
45 | }
46 |
47 | img {
48 | border-radius: 0.25em;
49 | display: block;
50 | }
51 |
--------------------------------------------------------------------------------
/examples/x-macos-pasteboard/solid-ui/src/main.tsx:
--------------------------------------------------------------------------------
1 | /* @refresh reload */
2 | import "./index.css";
3 | import { render } from "solid-js/web";
4 | import App from "./App.tsx";
5 |
6 | render(() => , document.getElementsByTagName("main")[0] as HTMLElement);
7 |
--------------------------------------------------------------------------------
/examples/x-macos-pasteboard/solid-ui/src/store/cas.ts:
--------------------------------------------------------------------------------
1 | import { createSignal } from "solid-js";
2 |
3 | export type CASStore = {
4 | get: (hash: string) => () => string | null;
5 | };
6 |
7 | export function createCAS(fetchContent: (hash: string) => Promise): CASStore {
8 | const cache = new Map string | null>();
9 |
10 | return {
11 | get(hash: string) {
12 | if (!cache.has(hash)) {
13 | const [content, setContent] = createSignal(null);
14 |
15 | // Cache the signal
16 | cache.set(hash, content);
17 |
18 | // Fetch the content and update the signal in the background
19 | fetchContent(hash)
20 | .then((data) => setContent(data))
21 | .catch((error) => {
22 | console.error("Failed to fetch content for hash:", error);
23 | });
24 | }
25 |
26 | // Return the signal for the content
27 | return cache.get(hash)!;
28 | },
29 | };
30 | }
31 |
--------------------------------------------------------------------------------
/examples/x-macos-pasteboard/solid-ui/src/store/index.ts:
--------------------------------------------------------------------------------
1 | import { createEffect, createMemo } from "solid-js";
2 | import { createStore } from "solid-js/store";
3 | import { Frame } from "./stream";
4 |
5 | export type StreamStore = { [key: string]: Frame[] };
6 |
7 | type StreamProps = {
8 | dataSignal: () => Frame | null;
9 | };
10 |
11 | export function useStore({ dataSignal }: StreamProps) {
12 | const [frames, setFrames] = createStore({});
13 |
14 | createEffect(() => {
15 | const frame = dataSignal();
16 | if (!frame) return;
17 |
18 | if (frame.topic !== "pb.recv" && frame.topic !== "content") return;
19 |
20 | const frameId = frame.meta?.updates ?? frame.id;
21 | setFrames(frameId, (existingFrames = []) => [frame, ...existingFrames]);
22 | });
23 |
24 | const index = createMemo(() => {
25 | return Object.keys(frames)
26 | .sort((a, b) => b.localeCompare(a))
27 | .map((id) => frames[id]);
28 | });
29 |
30 | return {
31 | index,
32 | };
33 | }
34 |
--------------------------------------------------------------------------------
/examples/x-macos-pasteboard/solid-ui/src/store/stream.ts:
--------------------------------------------------------------------------------
1 | import { createSignal, onCleanup, onMount } from "solid-js";
2 |
3 | export type Frame = {
4 | id: string;
5 | topic: string;
6 | hash: string;
7 | meta?: Record;
8 | };
9 |
10 | export function useFrameStream() {
11 | const [frame, setFrame] = createSignal(null);
12 |
13 | onMount(() => {
14 | const controller = new AbortController();
15 | const signal = controller.signal;
16 |
17 | const fetchData = async () => {
18 | const response = await fetch("/api?follow", { signal });
19 | const textStream = response.body!
20 | .pipeThrough(new TextDecoderStream())
21 | .pipeThrough(splitStream("\n"));
22 |
23 | const reader = textStream.getReader();
24 |
25 | while (true) {
26 | const { value, done } = await reader.read();
27 | if (done) break;
28 | if (value.trim()) {
29 | const json = JSON.parse(value);
30 | setFrame(json); // Update the signal with each new frame
31 | }
32 | }
33 |
34 | reader.releaseLock();
35 | };
36 |
37 | fetchData();
38 |
39 | onCleanup(() => {
40 | controller.abort();
41 | });
42 | });
43 |
44 | return frame;
45 | }
46 |
47 | // Utility function to split a stream by a delimiter
48 | function splitStream(delimiter: string) {
49 | let buffer = "";
50 | return new TransformStream({
51 | transform(chunk, controller) {
52 | buffer += chunk;
53 | const parts = buffer.split(delimiter);
54 | buffer = parts.pop()!;
55 | parts.forEach((part) => controller.enqueue(part));
56 | },
57 | flush(controller) {
58 | if (buffer) {
59 | controller.enqueue(buffer);
60 | }
61 | },
62 | });
63 | }
64 |
--------------------------------------------------------------------------------
/examples/x-macos-pasteboard/solid-ui/vite.config.ts:
--------------------------------------------------------------------------------
1 | import { defineConfig } from "vite";
2 | import deno from "@deno/vite-plugin";
3 | import solid from "vite-plugin-solid";
4 |
5 | // https://vite.dev/config/
6 | export default defineConfig({
7 | plugins: [deno(), solid()],
8 | server: {
9 | proxy: {
10 | "/api": {
11 | target: "http://localhost:3021",
12 | changeOrigin: true,
13 | rewrite: (path) => path.replace(/^\/api/, ""),
14 | },
15 | },
16 | },
17 | });
18 |
--------------------------------------------------------------------------------
/notes/how-to-release.md:
--------------------------------------------------------------------------------
1 | ```nushell
2 |
3 | # update version in Cargo.toml
4 | cargo b # to update Cargo.lock
5 |
6 | let PREVIOUS_RELEASE = git tag | lines | where {$in | str starts-with "v"} | sort | last
7 | let RELEASE = open Cargo.toml | get package.version
8 |
9 | # grab the raw commit messages between the previous release and now
10 | # create the release notes
11 | git log --format=%s $"($PREVIOUS_RELEASE)..HEAD" | vipe | save -f $"changes/($RELEASE).md"
12 | git add changes
13 |
14 | git commit -a -m $"chore: release ($RELEASE)"
15 | git push
16 |
17 | cargo publish
18 | cargo install cross-stream --locked
19 |
20 | rm ~/bin/xs
21 | brew uninstall cross-stream
22 | which xs # should be /Users/andy/.cargo/bin/xs
23 | # test the new version
24 |
25 | let pkgdir = $"cross-stream-($RELEASE)"
26 | let tarball = $"cross-stream-($RELEASE)-macos.tar.gz"
27 |
28 | mkdir $pkgdir
29 | cp /Users/andy/.cargo/bin/xs $pkgdir
30 | tar -czvf $tarball -C $pkgdir xs
31 |
32 | # git tag $"v($RELEASE)"
33 | # git push --tags
34 | # ^^ not needed, as the next line will create the tags -->
35 | gh release create $"v($RELEASE)" -F $"changes/($RELEASE).md" $tarball
36 |
37 | shasum -a 256 $tarball
38 |
39 | # update: git@github.com:cablehead/homebrew-tap.git
40 |
41 | brew install cablehead/tap/cross-stream
42 | which xs # should be /opt/homebrew/bin/xs
43 | # test the new version
44 | ```
45 |
--------------------------------------------------------------------------------
/notes/notes.md:
--------------------------------------------------------------------------------
1 | # xs
2 |
3 | ## Overview / Sketch
4 |
5 | An event stream store for personal, local-first use. Kinda like the
6 | [`sqlite3` cli](https://sqlite.org/cli.html), but specializing in the
7 | [event sourcing](https://martinfowler.com/eaaDev/EventSourcing.html) use case.
8 |
9 | 
10 |
11 | > "You don't so much run it, as poke _at_ it."
12 |
13 | Built with:
14 |
15 | - [fjall](https://github.com/fjall-rs/fjall): for indexing and metadata
16 | - [cacache](https://github.com/zkat/cacache-rs): for content (CAS)
17 | - [hyper](https://hyper.rs/guides/1/server/echo/): provides an HTTP/1.1 API over
18 | a local Unix domain socket for subscriptions, etc.
19 | - [nushell](https://www.nushell.sh): for scripting and
20 | [interop](https://utopia.rosano.ca/interoperable-visions/)
21 |
22 | ## Built-in Topics
23 |
24 | - `xs.start`: emitted when the server mounts the stream to expose an API
25 | - `xs.stop`: emitted when the server stops :: TODO
26 |
27 | - `xs.pulse`: (synthetic) a heartbeat event you can configure to be emitted every
28 | N seconds when in follow mode
29 |
30 | - `xs.threshold`: (synthetic) marks the boundary between
31 | replaying events and events that are newly arriving in real-time via a live
32 | subscription
33 |
34 | - `.spawn` :: spawn a generator
35 | - meta:: topic: string, duplex: bool
36 | - `.terminate`
37 |
38 | - `.register` :: register an event handler
39 | - meta:: run-from: start, tail, id?
40 | - `.unregister`
41 |
42 | ## Local socket HTTP API
43 |
44 | WIP, thoughts:
45 |
46 | - `/:topic` should probably be `/stream/:topic`
47 |
48 | ## API Endpoints
49 |
50 | ### GET
51 |
52 | - `/` - Pull the event stream
53 | - `/:id` - Pull a specific event by ID (where ID is a valid Scru128Id)
54 | - `/cas/:hash` - Pull the content addressed by `hash` (where hash is a valid ssri::Integrity)
55 |
56 | ### POST
57 |
58 | - `/:topic` - Append a new event to the stream for `topic`. The body of the POST
59 | will be stored in the CAS. You can also pass arbitrary JSON meta data using
60 | the `xs-meta` HTTP header.
61 | - `/pipe/:id` - Execute a script on a specific event. The ID should be a valid Scru128Id,
62 | and the body should contain the script to be executed.
63 |
64 | ## Features
65 |
66 | - event stream:
67 | - [x] append
68 | - [x] cat: last-id, follow, tail, threshold / heartbeat synthetic events
69 | - [x] get
70 | - [ ] last
71 | - [ ] first
72 | - [ ] next?
73 | - [ ] previous?
74 | - [x] cas, get
75 | - [x] ephemeral events / content
76 | - [ ] content can be chunked, to accomodate slow streams, e.g server sent events
77 | - [ ] secondary indexes for topics: the head of a topic can be used as a materialized view
78 | - process management: you can register snippets of Nushell on the event stream.
79 | server facilitates watching for updates + managing processes
80 | - [x] generators
81 | - [x] handlers
82 | - [x] builtin http server:
83 | - [x] You can optionally serve HTTP requests from your store. Requests are
84 | written to the event stream as `http.request` and then the connection
85 | watches the event stream for a `http.response`.
86 | - [x] You can register event handlers that subscribe to `http.request` events
87 | and emit `http.response` events.
88 | - Ability for a single xs process to serve many stores
89 | - so you generally run just one locally, using the systems local process
90 | manager, and then add and remove stores to serve via the event stream
91 |
92 | ## Path Traveled
93 |
94 | - [xs-3](https://github.com/cablehead/xs-3):
95 | [sled](https://github.com/spacejam/sled) index with
96 | [cacache](https://github.com/zkat/cacache-rs) CAS, no concurrency
97 | - [xs-0](https://github.com/cablehead/xs-0) original experiment.
98 | -[LMDB](http://www.lmdb.tech/doc/) combined index / content store (pre
99 | realizing the event primary content should be stored in a CAS)
100 | - Multi-process concurrent, but polling for subscribe
101 |
--------------------------------------------------------------------------------
/notes/overview.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/cablehead/xs/ea20566eddbbe76f9c4a6f1776b308b6db47d798/notes/overview.png
--------------------------------------------------------------------------------
/notes/screenshot.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/cablehead/xs/ea20566eddbbe76f9c4a6f1776b308b6db47d798/notes/screenshot.png
--------------------------------------------------------------------------------
/scripts/check.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | set -euo pipefail
4 |
5 | cargo fmt --check
6 | cargo clippy -- -D warnings
7 | cargo t
8 |
--------------------------------------------------------------------------------
/src/client/commands.rs:
--------------------------------------------------------------------------------
1 | use futures::StreamExt;
2 |
3 | use base64::Engine;
4 | use ssri::Integrity;
5 | use url::form_urlencoded;
6 |
7 | use http_body_util::{combinators::BoxBody, BodyExt, Empty, StreamBody};
8 | use hyper::body::Bytes;
9 | use hyper::Method;
10 | use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
11 | use tokio::sync::mpsc::Receiver;
12 | use tokio_util::io::ReaderStream;
13 |
14 | use super::request;
15 | use crate::store::{ReadOptions, TTL};
16 |
17 | pub async fn cat(
18 | addr: &str,
19 | options: ReadOptions,
20 | sse: bool,
21 | ) -> Result, Box> {
22 | // Convert any usize limit to u64
23 | let query = if options == ReadOptions::default() {
24 | None
25 | } else {
26 | Some(options.to_query_string())
27 | };
28 |
29 | let headers = if sse {
30 | Some(vec![(
31 | "Accept".to_string(),
32 | "text/event-stream".to_string(),
33 | )])
34 | } else {
35 | None
36 | };
37 |
38 | let res = request::request(addr, Method::GET, "", query.as_deref(), empty(), headers).await?;
39 |
40 | let (_parts, mut body) = res.into_parts();
41 | let (tx, rx) = tokio::sync::mpsc::channel(100);
42 |
43 | tokio::spawn(async move {
44 | while let Some(frame_result) = body.frame().await {
45 | match frame_result {
46 | Ok(frame) => {
47 | if let Ok(bytes) = frame.into_data() {
48 | if tx.send(bytes).await.is_err() {
49 | break;
50 | }
51 | }
52 | }
53 | Err(e) => {
54 | eprintln!("Error reading body: {}", e);
55 | break;
56 | }
57 | }
58 | }
59 | });
60 |
61 | Ok(rx)
62 | }
63 |
64 | pub async fn append(
65 | addr: &str,
66 | topic: &str,
67 | data: R,
68 | meta: Option<&serde_json::Value>,
69 | ttl: Option,
70 | context: Option<&str>,
71 | ) -> Result>
72 | where
73 | R: AsyncRead + Unpin + Send + 'static,
74 | {
75 | let mut params = Vec::new();
76 | if let Some(t) = ttl {
77 | let ttl_query = t.to_query();
78 | if let Some((k, v)) = ttl_query.split_once('=') {
79 | params.push((k.to_string(), v.to_string()));
80 | }
81 | }
82 | if let Some(c) = context {
83 | params.push(("context".to_string(), c.to_string()));
84 | }
85 |
86 | let query = if !params.is_empty() {
87 | Some(
88 | form_urlencoded::Serializer::new(String::new())
89 | .extend_pairs(params)
90 | .finish(),
91 | )
92 | } else {
93 | None
94 | };
95 |
96 | let reader_stream = ReaderStream::new(data);
97 | let mapped_stream = reader_stream.map(|result| {
98 | result
99 | .map(hyper::body::Frame::data)
100 | .map_err(|e| Box::new(e) as Box)
101 | });
102 | let body = StreamBody::new(mapped_stream);
103 |
104 | let headers = meta.map(|meta_value| {
105 | let json_string = serde_json::to_string(meta_value).unwrap();
106 | let encoded = base64::prelude::BASE64_STANDARD.encode(json_string);
107 | vec![("xs-meta".to_string(), encoded)]
108 | });
109 |
110 | let res = request::request(addr, Method::POST, topic, query.as_deref(), body, headers).await?;
111 | let body = res.collect().await?.to_bytes();
112 | Ok(body)
113 | }
114 |
115 | pub async fn cas_get(
116 | addr: &str,
117 | integrity: Integrity,
118 | writer: &mut W,
119 | ) -> Result<(), Box>
120 | where
121 | W: AsyncWrite + Unpin,
122 | {
123 | let parts = super::types::RequestParts::parse(addr, &format!("cas/{}", integrity), None)?;
124 |
125 | match parts.connection {
126 | super::types::ConnectionKind::Unix(path) => {
127 | // Direct CAS access for local path
128 | let store_path = path.parent().unwrap_or(&path).to_path_buf();
129 | let cas_path = store_path.join("cacache");
130 | let mut reader = cacache::Reader::open_hash(&cas_path, integrity).await?;
131 | tokio::io::copy(&mut reader, writer).await?;
132 | writer.flush().await?;
133 | Ok(())
134 | }
135 | _ => {
136 | // Remote HTTP access
137 | let res = request::request(
138 | addr,
139 | Method::GET,
140 | &format!("cas/{}", integrity),
141 | None,
142 | empty(),
143 | None,
144 | )
145 | .await?;
146 | let mut body = res.into_body();
147 |
148 | while let Some(frame) = body.frame().await {
149 | let frame = frame?;
150 | if let Ok(chunk) = frame.into_data() {
151 | writer.write_all(&chunk).await?;
152 | }
153 | }
154 |
155 | writer.flush().await?;
156 | Ok(())
157 | }
158 | }
159 | }
160 |
161 | pub async fn cas_post(
162 | addr: &str,
163 | data: R,
164 | ) -> Result>
165 | where
166 | R: AsyncRead + Unpin + Send + 'static,
167 | {
168 | let reader_stream = ReaderStream::new(data);
169 | let mapped_stream = reader_stream.map(|result| {
170 | result
171 | .map(hyper::body::Frame::data)
172 | .map_err(|e| Box::new(e) as Box)
173 | });
174 | let body = StreamBody::new(mapped_stream);
175 |
176 | let res = request::request(addr, Method::POST, "cas", None, body, None).await?;
177 | let body = res.collect().await?.to_bytes();
178 | Ok(body)
179 | }
180 |
181 | pub async fn get(addr: &str, id: &str) -> Result> {
182 | let res = request::request(addr, Method::GET, id, None, empty(), None).await?;
183 | let body = res.collect().await?.to_bytes();
184 | Ok(body)
185 | }
186 |
187 | pub async fn remove(addr: &str, id: &str) -> Result<(), Box> {
188 | let _ = request::request(addr, Method::DELETE, id, None, empty(), None).await?;
189 | Ok(())
190 | }
191 |
192 | pub async fn head(
193 | addr: &str,
194 | topic: &str,
195 | follow: bool,
196 | context: Option<&str>,
197 | ) -> Result<(), Box> {
198 | let mut params = Vec::new();
199 | if follow {
200 | params.push(("follow", "true".to_string()));
201 | }
202 | if let Some(c) = context {
203 | params.push(("context", c.to_string()));
204 | }
205 |
206 | let query = if !params.is_empty() {
207 | Some(
208 | form_urlencoded::Serializer::new(String::new())
209 | .extend_pairs(params)
210 | .finish(),
211 | )
212 | } else {
213 | None
214 | };
215 |
216 | let res = request::request(
217 | addr,
218 | Method::GET,
219 | &format!("head/{}", topic),
220 | query.as_deref(),
221 | empty(),
222 | None,
223 | )
224 | .await?;
225 |
226 | let mut body = res.into_body();
227 | let mut stdout = tokio::io::stdout();
228 |
229 | while let Some(frame) = body.frame().await {
230 | let frame = frame?;
231 | if let Ok(chunk) = frame.into_data() {
232 | stdout.write_all(&chunk).await?;
233 | }
234 | }
235 | stdout.flush().await?;
236 | Ok(())
237 | }
238 |
239 | pub async fn import(
240 | addr: &str,
241 | data: R,
242 | ) -> Result>
243 | where
244 | R: AsyncRead + Unpin + Send + 'static,
245 | {
246 | let reader_stream = ReaderStream::new(data);
247 | let mapped_stream = reader_stream.map(|result| {
248 | result
249 | .map(hyper::body::Frame::data)
250 | .map_err(|e| Box::new(e) as Box)
251 | });
252 | let body = StreamBody::new(mapped_stream);
253 |
254 | let res = request::request(addr, Method::POST, "import", None, body, None).await?;
255 | let body = res.collect().await?.to_bytes();
256 | Ok(body)
257 | }
258 |
259 | pub async fn version(addr: &str) -> Result> {
260 | match request::request(addr, Method::GET, "version", None, empty(), None).await {
261 | Ok(res) => {
262 | let body = res.collect().await?.to_bytes();
263 | Ok(body)
264 | }
265 | Err(e) => {
266 | // this was the version before the /version endpoint was added
267 | if e.to_string().contains("404 Not Found") {
268 | Ok(Bytes::from(r#"{"version":"0.0.9"}"#))
269 | } else {
270 | Err(e)
271 | }
272 | }
273 | }
274 | }
275 |
276 | fn empty() -> BoxBody> {
277 | Empty::::new()
278 | .map_err(|never| match never {})
279 | .boxed()
280 | }
281 |
--------------------------------------------------------------------------------
/src/client/connect.rs:
--------------------------------------------------------------------------------
1 | use crate::listener::AsyncReadWriteBox;
2 | use rustls::pki_types::ServerName;
3 | use rustls::ClientConfig;
4 | use rustls::RootCertStore;
5 | use std::sync::Arc;
6 | use tokio::net::{TcpStream, UnixStream};
7 | use tokio_rustls::TlsConnector;
8 |
9 | use super::types::{BoxError, ConnectionKind, RequestParts};
10 |
11 | async fn create_tls_connector() -> Result {
12 | let mut root_store = RootCertStore::empty();
13 | root_store.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned());
14 | let config = ClientConfig::builder()
15 | .with_root_certificates(root_store)
16 | .with_no_client_auth();
17 | Ok(TlsConnector::from(Arc::new(config)))
18 | }
19 |
20 | pub async fn connect(parts: &RequestParts) -> Result {
21 | match &parts.connection {
22 | ConnectionKind::Unix(path) => {
23 | let stream = UnixStream::connect(path).await?;
24 | Ok(Box::new(stream))
25 | }
26 | ConnectionKind::Tcp { host, port } => {
27 | let stream = TcpStream::connect((host.as_str(), *port)).await?;
28 | Ok(Box::new(stream))
29 | }
30 | ConnectionKind::Tls { host, port } => {
31 | let tcp_stream = TcpStream::connect((host.as_str(), *port)).await?;
32 | let connector = create_tls_connector().await?;
33 | let server_name = ServerName::try_from(host.clone())?; // Clone the host string
34 | let tls_stream = connector.connect(server_name, tcp_stream).await?;
35 | Ok(Box::new(tls_stream))
36 | }
37 | }
38 | }
39 |
--------------------------------------------------------------------------------
/src/client/mod.rs:
--------------------------------------------------------------------------------
1 | mod commands;
2 | mod connect;
3 | mod request;
4 | mod types;
5 |
6 | pub use self::commands::{append, cas_get, cas_post, cat, get, head, import, remove, version};
7 |
--------------------------------------------------------------------------------
/src/client/request.rs:
--------------------------------------------------------------------------------
1 | use http_body_util::BodyExt;
2 | use hyper::{Method, Request};
3 | use hyper_util::rt::TokioIo;
4 |
5 | use super::connect::connect;
6 | use super::types::{BoxError, RequestParts};
7 |
8 | pub async fn request(
9 | addr: &str,
10 | method: Method,
11 | path: &str,
12 | query: Option<&str>,
13 | body: B,
14 | headers: Option>,
15 | ) -> Result, BoxError>
16 | where
17 | B: hyper::body::Body + Send + 'static,
18 | B::Error: Into + Send,
19 | {
20 | let parts = RequestParts::parse(addr, path, query)?;
21 | let stream = connect(&parts).await?;
22 | let io = TokioIo::new(stream);
23 | let (mut sender, conn) = hyper::client::conn::http1::handshake(io).await?;
24 |
25 | tokio::spawn(async move {
26 | if let Err(e) = conn.await {
27 | eprintln!("Connection error: {}", e);
28 | }
29 | });
30 |
31 | let mut builder = Request::builder()
32 | .method(method)
33 | .uri(parts.uri)
34 | .header(hyper::header::USER_AGENT, "xs/0.1")
35 | .header(hyper::header::ACCEPT, "*/*");
36 |
37 | if let Some(host) = parts.host {
38 | builder = builder.header(hyper::header::HOST, host);
39 | }
40 | if let Some(auth) = parts.authorization {
41 | builder = builder.header(hyper::header::AUTHORIZATION, auth);
42 | }
43 |
44 | if let Some(extra_headers) = headers {
45 | for (name, value) in extra_headers {
46 | builder = builder.header(name, value);
47 | }
48 | }
49 |
50 | let req = builder.body(body)?;
51 | let res = sender.send_request(req).await?;
52 |
53 | // Handle non-OK responses
54 | if res.status() != hyper::StatusCode::OK && res.status() != hyper::StatusCode::NO_CONTENT {
55 | let status = res.status();
56 | let body = res.collect().await?.to_bytes();
57 | return Err(format!("{}:: {}", status, String::from_utf8_lossy(&body)).into());
58 | }
59 |
60 | Ok(res)
61 | }
62 |
--------------------------------------------------------------------------------
/src/client/types.rs:
--------------------------------------------------------------------------------
1 | use base64::prelude::*;
2 |
3 | pub type BoxError = Box;
4 |
5 | #[derive(Debug, PartialEq)]
6 | pub enum ConnectionKind {
7 | Unix(std::path::PathBuf),
8 | Tcp { host: String, port: u16 },
9 | Tls { host: String, port: u16 },
10 | }
11 |
12 | #[derive(Debug, PartialEq)]
13 | pub struct RequestParts {
14 | pub uri: String,
15 | pub host: Option,
16 | pub authorization: Option,
17 | pub connection: ConnectionKind,
18 | }
19 |
20 | impl RequestParts {
21 | pub fn parse(
22 | addr: &str,
23 | path: &str,
24 | query: Option<&str>,
25 | ) -> Result> {
26 | // Unix socket case
27 | if addr.starts_with('/') || addr.starts_with('.') {
28 | let socket_path = if std::path::Path::new(addr).is_dir() {
29 | std::path::Path::new(addr).join("sock")
30 | } else {
31 | std::path::Path::new(addr).to_path_buf()
32 | };
33 |
34 | return Ok(RequestParts {
35 | uri: if let Some(q) = query {
36 | format!("http://localhost/{}?{}", path, q)
37 | } else {
38 | format!("http://localhost/{}", path)
39 | },
40 | host: None,
41 | authorization: None,
42 | connection: ConnectionKind::Unix(socket_path),
43 | });
44 | }
45 |
46 | // Normalize URL
47 | let addr = if addr.starts_with(':') {
48 | format!("http://127.0.0.1{}", addr)
49 | } else if !addr.contains("://") {
50 | format!("http://{}", addr)
51 | } else {
52 | addr.to_string()
53 | };
54 |
55 | let url = url::Url::parse(&addr)?;
56 | let scheme = url.scheme();
57 | let host = url.host_str().ok_or("Missing host")?.to_string();
58 | let port = url
59 | .port()
60 | .unwrap_or(if scheme == "https" { 443 } else { 80 });
61 | let port_str = if (scheme == "http" && port == 80) || (scheme == "https" && port == 443) {
62 | "".to_string()
63 | } else {
64 | format!(":{}", port)
65 | };
66 |
67 | // Build clean request URI (no auth)
68 | let uri = if let Some(q) = query {
69 | format!("{}://{}{}/{}?{}", scheme, host, port_str, path, q)
70 | } else {
71 | format!("{}://{}{}/{}", scheme, host, port_str, path)
72 | };
73 |
74 | // Set auth if present
75 | let authorization = if let Some(password) = url.password() {
76 | let credentials = format!("{}:{}", url.username(), password);
77 | Some(format!(
78 | "Basic {}",
79 | base64::prelude::BASE64_STANDARD.encode(credentials)
80 | ))
81 | } else if !url.username().is_empty() {
82 | let credentials = format!("{}:", url.username());
83 | Some(format!(
84 | "Basic {}",
85 | base64::prelude::BASE64_STANDARD.encode(credentials)
86 | ))
87 | } else {
88 | None
89 | };
90 |
91 | Ok(RequestParts {
92 | uri,
93 | host: Some(format!("{}{}", host, port_str)),
94 | authorization,
95 | connection: if scheme == "https" {
96 | ConnectionKind::Tls { host, port }
97 | } else {
98 | ConnectionKind::Tcp { host, port }
99 | },
100 | })
101 | }
102 | }
103 |
104 | #[cfg(test)]
105 | mod tests {
106 | use super::*;
107 |
108 | #[test]
109 | fn test_unix_socket() {
110 | let parts = RequestParts::parse("./store", "foo", None).unwrap();
111 | assert_eq!(parts.uri, "http://localhost/foo");
112 | assert_eq!(parts.host, None);
113 | assert_eq!(parts.authorization, None);
114 | }
115 |
116 | #[test]
117 | fn test_port_only() {
118 | let parts = RequestParts::parse(":8080", "bar", Some("q=1")).unwrap();
119 | assert_eq!(parts.uri, "http://127.0.0.1:8080/bar?q=1");
120 | assert_eq!(parts.host, Some("127.0.0.1:8080".to_string()));
121 | assert_eq!(parts.authorization, None);
122 | }
123 |
124 | #[test]
125 | fn test_https_url_with_auth() {
126 | let parts = RequestParts::parse("https://user:pass@example.com:400", "", None).unwrap();
127 | assert_eq!(parts.uri, "https://example.com:400/");
128 | assert_eq!(parts.host, Some("example.com:400".to_string()));
129 | assert_eq!(parts.authorization, Some("Basic dXNlcjpwYXNz".to_string()));
130 | }
131 | }
132 |
--------------------------------------------------------------------------------
/src/commands/mod.rs:
--------------------------------------------------------------------------------
1 | mod serve;
2 |
3 | #[cfg(test)]
4 | mod tests;
5 |
6 | pub use serve::serve;
7 |
--------------------------------------------------------------------------------
/src/error.rs:
--------------------------------------------------------------------------------
1 | pub type Error = Box;
2 |
--------------------------------------------------------------------------------
/src/generators/mod.rs:
--------------------------------------------------------------------------------
1 | mod generator;
2 | mod serve;
3 |
4 | pub use generator::{
5 | spawn as spawn_generator_loop, GeneratorEventKind, GeneratorLoop, GeneratorScriptOptions,
6 | StopReason, Task,
7 | };
8 |
9 | #[cfg(test)]
10 | mod tests;
11 |
12 | pub use serve::serve;
13 |
--------------------------------------------------------------------------------
/src/generators/serve.rs:
--------------------------------------------------------------------------------
1 | use std::collections::HashMap;
2 |
3 | use scru128::Scru128Id;
4 | use serde_json::json;
5 | use tokio::task::JoinHandle;
6 |
7 | use crate::generators::generator;
8 | use crate::nu;
9 | use crate::store::{FollowOption, Frame, ReadOptions, Store};
10 |
11 | async fn try_start_task(
12 | topic: &str,
13 | frame: &Frame,
14 | active: &mut HashMap<(String, Scru128Id), JoinHandle<()>>,
15 | engine: &nu::Engine,
16 | store: &Store,
17 | ) {
18 | if let Err(e) =
19 | handle_spawn_event(topic, frame.clone(), active, engine.clone(), store.clone()).await
20 | {
21 | let meta = json!({
22 | "source_id": frame.id.to_string(),
23 | "reason": e.to_string()
24 | });
25 |
26 | if let Err(e) = store.append(
27 | Frame::builder(format!("{}.parse.error", topic), frame.context_id)
28 | .meta(meta)
29 | .build(),
30 | ) {
31 | tracing::error!("Error appending error frame: {}", e);
32 | }
33 | }
34 | }
35 |
36 | async fn handle_spawn_event(
37 | topic: &str,
38 | frame: Frame,
39 | active: &mut HashMap<(String, Scru128Id), JoinHandle<()>>,
40 | engine: nu::Engine,
41 | store: Store,
42 | ) -> Result<(), Box> {
43 | let key = (topic.to_string(), frame.context_id);
44 | if let Some(handle) = active.get(&key) {
45 | if handle.is_finished() {
46 | active.remove(&key);
47 | } else {
48 | // A generator for this topic/context is already running. Ignore the
49 | // new spawn frame; the running generator will handle it as a hot
50 | // reload.
51 | return Ok(());
52 | }
53 | }
54 |
55 | let handle = generator::spawn(store.clone(), engine.clone(), frame);
56 | active.insert(key, handle);
57 | Ok(())
58 | }
59 |
60 | pub async fn serve(
61 | store: Store,
62 | engine: nu::Engine,
63 | ) -> Result<(), Box> {
64 | let options = ReadOptions::builder().follow(FollowOption::On).build();
65 | let mut recver = store.read(options).await;
66 |
67 | let mut active: HashMap<(String, Scru128Id), JoinHandle<()>> = HashMap::new();
68 | let mut compacted: HashMap<(String, Scru128Id), Frame> = HashMap::new();
69 |
70 | while let Some(frame) = recver.recv().await {
71 | if frame.topic == "xs.threshold" {
72 | break;
73 | }
74 | if frame.topic.ends_with(".spawn") || frame.topic.ends_with(".parse.error") {
75 | if let Some(prefix) = frame
76 | .topic
77 | .strip_suffix(".parse.error")
78 | .or_else(|| frame.topic.strip_suffix(".spawn"))
79 | {
80 | compacted.insert((prefix.to_string(), frame.context_id), frame);
81 | }
82 | } else if let Some(prefix) = frame.topic.strip_suffix(".terminate") {
83 | compacted.remove(&(prefix.to_string(), frame.context_id));
84 | }
85 | }
86 |
87 | for ((topic, _), frame) in &compacted {
88 | if frame.topic.ends_with(".spawn") {
89 | try_start_task(topic, frame, &mut active, &engine, &store).await;
90 | }
91 | }
92 |
93 | while let Some(frame) = recver.recv().await {
94 | if let Some(prefix) = frame.topic.strip_suffix(".spawn") {
95 | try_start_task(prefix, &frame, &mut active, &engine, &store).await;
96 | continue;
97 | }
98 |
99 | if let Some(_prefix) = frame.topic.strip_suffix(".parse.error") {
100 | // parse.error frames are informational; ignore them
101 | continue;
102 | }
103 |
104 | if let Some(prefix) = frame.topic.strip_suffix(".shutdown") {
105 | active.remove(&(prefix.to_string(), frame.context_id));
106 | continue;
107 | }
108 | }
109 |
110 | Ok(())
111 | }
112 |
--------------------------------------------------------------------------------
/src/handlers/mod.rs:
--------------------------------------------------------------------------------
1 | mod handler;
2 | mod serve;
3 | #[cfg(test)]
4 | mod tests;
5 |
6 | pub use handler::Handler;
7 | pub use serve::serve;
8 |
--------------------------------------------------------------------------------
/src/handlers/serve.rs:
--------------------------------------------------------------------------------
1 | use std::collections::HashMap;
2 |
3 | use crate::handlers::Handler;
4 | use crate::nu;
5 | use crate::nu::commands;
6 | use crate::store::{FollowOption, Frame, ReadOptions, Store};
7 |
8 | async fn start_handler(
9 | frame: &Frame,
10 | store: &Store,
11 | engine: &nu::Engine,
12 | topic: &str,
13 | ) -> Result<(), Box> {
14 | match Handler::from_frame(frame, store, engine.clone()).await {
15 | Ok(handler) => {
16 | handler.spawn(store.clone()).await?;
17 | Ok(())
18 | }
19 | Err(err) => {
20 | let _ = store.append(
21 | Frame::builder(format!("{}.unregistered", topic), frame.context_id)
22 | .meta(serde_json::json!({
23 | "handler_id": frame.id.to_string(),
24 | "error": err.to_string(),
25 | }))
26 | .build(),
27 | );
28 | Ok(())
29 | }
30 | }
31 | }
32 |
33 | #[derive(Debug)]
34 | struct TopicState {
35 | register_frame: Frame,
36 | handler_id: String,
37 | }
38 |
39 | pub async fn serve(
40 | store: Store,
41 | mut engine: nu::Engine,
42 | ) -> Result<(), Box> {
43 | engine.add_commands(vec![
44 | Box::new(commands::cas_command::CasCommand::new(store.clone())),
45 | Box::new(commands::get_command::GetCommand::new(store.clone())),
46 | Box::new(commands::remove_command::RemoveCommand::new(store.clone())),
47 | ])?;
48 | engine.add_alias(".rm", ".remove")?;
49 |
50 | let options = ReadOptions::builder().follow(FollowOption::On).build();
51 |
52 | let mut recver = store.read(options).await;
53 | let mut topic_states: HashMap<(String, scru128::Scru128Id), TopicState> = HashMap::new();
54 |
55 | // Process historical frames until threshold
56 | while let Some(frame) = recver.recv().await {
57 | if frame.topic == "xs.threshold" {
58 | break;
59 | }
60 |
61 | // Extract base topic and suffix
62 | if let Some((topic, suffix)) = frame.topic.rsplit_once('.') {
63 | match suffix {
64 | "register" => {
65 | // Store new registration
66 | topic_states.insert(
67 | (topic.to_string(), frame.context_id),
68 | TopicState {
69 | register_frame: frame.clone(),
70 | handler_id: frame.id.to_string(),
71 | },
72 | );
73 | }
74 | "unregister" | "unregistered" => {
75 | // Only remove if handler_id matches
76 | if let Some(meta) = &frame.meta {
77 | if let Some(handler_id) = meta.get("handler_id").and_then(|v| v.as_str()) {
78 | let key = (topic.to_string(), frame.context_id);
79 | if let Some(state) = topic_states.get(&key) {
80 | if state.handler_id == handler_id {
81 | topic_states.remove(&key);
82 | }
83 | }
84 | }
85 | }
86 | }
87 | _ => {}
88 | }
89 | }
90 | }
91 |
92 | // Process all retained registrations ordered by frame ID
93 | let mut ordered_states: Vec<_> = topic_states.values().collect();
94 | ordered_states.sort_by_key(|state| state.register_frame.id);
95 |
96 | for state in ordered_states {
97 | if let Some(topic) = state.register_frame.topic.strip_suffix(".register") {
98 | start_handler(&state.register_frame, &store, &engine, topic).await?;
99 | }
100 | }
101 |
102 | // Continue processing new frames
103 | while let Some(frame) = recver.recv().await {
104 | if let Some(topic) = frame.topic.strip_suffix(".register") {
105 | start_handler(&frame, &store, &engine, topic).await?;
106 | }
107 | }
108 |
109 | Ok(())
110 | }
111 |
--------------------------------------------------------------------------------
/src/lib.rs:
--------------------------------------------------------------------------------
1 | pub mod api;
2 | pub mod client;
3 | pub mod commands;
4 | pub mod error;
5 | pub mod generators;
6 | pub mod handlers;
7 | pub mod listener;
8 | pub mod nu;
9 | pub mod store;
10 | pub mod trace;
11 |
--------------------------------------------------------------------------------
/src/listener.rs:
--------------------------------------------------------------------------------
1 | use std::io;
2 |
3 | use tokio::io::{AsyncRead, AsyncWrite};
4 | use tokio::net::{TcpListener, UnixListener};
5 | #[cfg(test)]
6 | use tokio::net::{TcpStream, UnixStream};
7 |
8 | pub trait AsyncReadWrite: AsyncRead + AsyncWrite {}
9 |
10 | impl AsyncReadWrite for T {}
11 |
12 | pub type AsyncReadWriteBox = Box;
13 |
14 | pub enum Listener {
15 | Tcp(TcpListener),
16 | Unix(UnixListener),
17 | }
18 |
19 | impl Listener {
20 | pub async fn accept(
21 | &mut self,
22 | ) -> io::Result<(AsyncReadWriteBox, Option)> {
23 | match self {
24 | Listener::Tcp(listener) => {
25 | let (stream, addr) = listener.accept().await?;
26 | Ok((Box::new(stream), Some(addr)))
27 | }
28 | Listener::Unix(listener) => {
29 | let (stream, _) = listener.accept().await?;
30 | Ok((Box::new(stream), None))
31 | }
32 | }
33 | }
34 |
35 | pub async fn bind(addr: &str) -> io::Result {
36 | if addr.starts_with('/') || addr.starts_with('.') {
37 | // attempt to remove the socket unconditionally
38 | let _ = std::fs::remove_file(addr);
39 | let listener = UnixListener::bind(addr)?;
40 | Ok(Listener::Unix(listener))
41 | } else {
42 | let mut addr = addr.to_owned();
43 | if addr.starts_with(':') {
44 | addr = format!("127.0.0.1{}", addr);
45 | };
46 | let listener = TcpListener::bind(addr).await?;
47 | Ok(Listener::Tcp(listener))
48 | }
49 | }
50 |
51 | #[cfg(test)]
52 | pub async fn connect(&self) -> io::Result {
53 | match self {
54 | Listener::Tcp(listener) => {
55 | let stream = TcpStream::connect(listener.local_addr()?).await?;
56 | Ok(Box::new(stream))
57 | }
58 | Listener::Unix(listener) => {
59 | let stream =
60 | UnixStream::connect(listener.local_addr()?.as_pathname().unwrap()).await?;
61 | Ok(Box::new(stream))
62 | }
63 | }
64 | }
65 | }
66 |
67 | impl std::fmt::Display for Listener {
68 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
69 | match self {
70 | Listener::Tcp(listener) => {
71 | let addr = listener.local_addr().unwrap();
72 | write!(f, "{}:{}", addr.ip(), addr.port())
73 | }
74 | Listener::Unix(listener) => {
75 | let addr = listener.local_addr().unwrap();
76 | let path = addr.as_pathname().unwrap();
77 | write!(f, "{}", path.display())
78 | }
79 | }
80 | }
81 | }
82 |
83 | #[cfg(test)]
84 | mod tests {
85 | use super::*;
86 |
87 | use tokio::io::AsyncReadExt;
88 | use tokio::io::AsyncWriteExt;
89 |
90 | async fn exercise_listener(addr: &str) {
91 | let mut listener = Listener::bind(addr).await.unwrap();
92 | let mut client = listener.connect().await.unwrap();
93 |
94 | let (mut serve, _) = listener.accept().await.unwrap();
95 | let want = b"Hello from server!";
96 | serve.write_all(want).await.unwrap();
97 | drop(serve);
98 |
99 | let mut got = Vec::new();
100 | client.read_to_end(&mut got).await.unwrap();
101 | assert_eq!(want.to_vec(), got);
102 | }
103 |
104 | #[tokio::test]
105 | async fn test_bind_tcp() {
106 | exercise_listener(":0").await;
107 | }
108 |
109 | #[tokio::test]
110 | async fn test_bind_unix() {
111 | let temp_dir = tempfile::tempdir().unwrap();
112 | let path = temp_dir.path().join("test.sock");
113 | let path = path.to_str().unwrap();
114 | exercise_listener(path).await;
115 | }
116 | }
117 |
--------------------------------------------------------------------------------
/src/nu/commands/append_command.rs:
--------------------------------------------------------------------------------
1 | use nu_engine::CallExt;
2 | use nu_protocol::engine::{Call, Command, EngineState, Stack};
3 | use nu_protocol::{Category, PipelineData, ShellError, Signature, SyntaxShape, Type, Value};
4 |
5 | use serde_json::Value as JsonValue;
6 |
7 | use crate::nu::util;
8 | use crate::store::{Frame, Store, TTL};
9 |
10 | #[derive(Clone)]
11 | pub struct AppendCommand {
12 | store: Store,
13 | context_id: scru128::Scru128Id,
14 | base_meta: JsonValue,
15 | }
16 |
17 | impl AppendCommand {
18 | pub fn new(store: Store, context_id: scru128::Scru128Id, base_meta: JsonValue) -> Self {
19 | Self {
20 | store,
21 | context_id,
22 | base_meta,
23 | }
24 | }
25 | }
26 |
27 | impl Command for AppendCommand {
28 | fn name(&self) -> &str {
29 | ".append"
30 | }
31 |
32 | fn signature(&self) -> Signature {
33 | Signature::build(".append")
34 | .input_output_types(vec![(Type::Any, Type::Any)])
35 | .required("topic", SyntaxShape::String, "this clip's topic")
36 | .named(
37 | "meta",
38 | SyntaxShape::Record(vec![]),
39 | "arbitrary metadata",
40 | None,
41 | )
42 | .named(
43 | "ttl",
44 | SyntaxShape::String,
45 | r#"TTL specification: 'forever', 'ephemeral', 'time:', or 'head:'"#,
46 | None,
47 | )
48 | .named(
49 | "context",
50 | SyntaxShape::String,
51 | "context ID (defaults to system context)",
52 | None,
53 | )
54 | .category(Category::Experimental)
55 | }
56 |
57 | fn description(&self) -> &str {
58 | "Writes its input to the CAS and then appends a frame with a hash of this content to the given topic on the stream."
59 | }
60 |
61 | fn run(
62 | &self,
63 | engine_state: &EngineState,
64 | stack: &mut Stack,
65 | call: &Call,
66 | input: PipelineData,
67 | ) -> Result {
68 | let span = call.head;
69 |
70 | let store = self.store.clone();
71 |
72 | let topic: String = call.req(engine_state, stack, 0)?;
73 |
74 | // Get user-supplied metadata and convert to JSON
75 | let user_meta: Option = call.get_flag(engine_state, stack, "meta")?;
76 | let mut final_meta = self.base_meta.clone(); // Start with base metadata
77 |
78 | // Merge user metadata if provided
79 | if let Some(user_value) = user_meta {
80 | let user_json = util::value_to_json(&user_value);
81 | if let JsonValue::Object(mut base_obj) = final_meta {
82 | if let JsonValue::Object(user_obj) = user_json {
83 | base_obj.extend(user_obj); // Merge user metadata into base
84 | final_meta = JsonValue::Object(base_obj);
85 | } else {
86 | return Err(ShellError::TypeMismatch {
87 | err_message: "Meta must be a record".to_string(),
88 | span: call.span(),
89 | });
90 | }
91 | }
92 | }
93 |
94 | let ttl: Option = call.get_flag(engine_state, stack, "ttl")?;
95 | let ttl = match ttl {
96 | Some(ttl_str) => Some(TTL::from_query(Some(&format!("ttl={}", ttl_str))).map_err(
97 | |e| ShellError::TypeMismatch {
98 | err_message: format!("Invalid TTL value: {}. {}", ttl_str, e),
99 | span: call.span(),
100 | },
101 | )?),
102 | None => None,
103 | };
104 |
105 | let hash = util::write_pipeline_to_cas(input, &store, span).map_err(|boxed| *boxed)?;
106 | let context_str: Option = call.get_flag(engine_state, stack, "context")?;
107 | let context_id = context_str
108 | .map(|ctx| ctx.parse::())
109 | .transpose()
110 | .map_err(|e| ShellError::GenericError {
111 | error: "Invalid context ID".into(),
112 | msg: e.to_string(),
113 | span: Some(call.head),
114 | help: None,
115 | inner: vec![],
116 | })?
117 | .unwrap_or(self.context_id);
118 |
119 | let frame = store.append(
120 | Frame::builder(topic, context_id)
121 | .maybe_hash(hash)
122 | .meta(final_meta)
123 | .maybe_ttl(ttl)
124 | .build(),
125 | )?;
126 |
127 | Ok(PipelineData::Value(
128 | util::frame_to_value(&frame, span),
129 | None,
130 | ))
131 | }
132 | }
133 |
--------------------------------------------------------------------------------
/src/nu/commands/append_command_buffered.rs:
--------------------------------------------------------------------------------
1 | use std::sync::{Arc, Mutex};
2 |
3 | use nu_engine::CallExt;
4 | use nu_protocol::engine::{Call, Command, EngineState, Stack};
5 | use nu_protocol::{Category, PipelineData, ShellError, Signature, SyntaxShape, Type, Value};
6 |
7 | use crate::nu::util::value_to_json;
8 | use crate::store::{Frame, Store, TTL};
9 |
10 | #[derive(Clone)]
11 | pub struct AppendCommand {
12 | output: Arc>>,
13 | store: Store,
14 | }
15 |
16 | impl AppendCommand {
17 | pub fn new(store: Store, output: Arc>>) -> Self {
18 | Self { output, store }
19 | }
20 | }
21 |
22 | impl Command for AppendCommand {
23 | fn name(&self) -> &str {
24 | ".append"
25 | }
26 |
27 | fn signature(&self) -> Signature {
28 | Signature::build(".append")
29 | .input_output_types(vec![(Type::Any, Type::Any)])
30 | .required("topic", SyntaxShape::String, "this clip's topic")
31 | .named(
32 | "meta",
33 | SyntaxShape::Record(vec![]),
34 | "arbitrary metadata",
35 | None,
36 | )
37 | .named(
38 | "ttl",
39 | SyntaxShape::String,
40 | r#"TTL specification: 'forever', 'ephemeral', 'time:', or 'head:'"#,
41 | None,
42 | )
43 | .named(
44 | "context",
45 | SyntaxShape::String,
46 | "context ID (defaults to system context)",
47 | None,
48 | )
49 | .category(Category::Experimental)
50 | }
51 |
52 | fn description(&self) -> &str {
53 | "Writes its input to the CAS and buffers a frame for later batch processing. The frame will include the content hash, any provided metadata and TTL settings. Meant for use with handlers that need to batch multiple appends."
54 | }
55 |
56 | fn run(
57 | &self,
58 | engine_state: &EngineState,
59 | stack: &mut Stack,
60 | call: &Call,
61 | input: PipelineData,
62 | ) -> Result {
63 | let span = call.head;
64 |
65 | let topic: String = call.req(engine_state, stack, 0)?;
66 | let meta: Option = call.get_flag(engine_state, stack, "meta")?;
67 | let ttl_str: Option = call.get_flag(engine_state, stack, "ttl")?;
68 |
69 | let ttl = ttl_str
70 | .map(|s| TTL::from_query(Some(&format!("ttl={}", s))))
71 | .transpose()
72 | .map_err(|e| ShellError::GenericError {
73 | error: "Invalid TTL format".into(),
74 | msg: e.to_string(),
75 | span: Some(span),
76 | help: Some("TTL must be one of: 'forever', 'ephemeral', 'time:', or 'head:'".into()),
77 | inner: vec![],
78 | })?;
79 |
80 | let input_value = input.into_value(span)?;
81 |
82 | let hash = crate::nu::util::write_pipeline_to_cas(
83 | PipelineData::Value(input_value.clone(), None),
84 | &self.store,
85 | span,
86 | )
87 | .map_err(|boxed| *boxed)?;
88 |
89 | let context_str: Option = call.get_flag(engine_state, stack, "context")?;
90 | let context_id = if let Some(ctx) = context_str {
91 | ctx.parse::()
92 | .map_err(|e| ShellError::GenericError {
93 | error: "Invalid context ID".into(),
94 | msg: e.to_string(),
95 | span: Some(call.head),
96 | help: None,
97 | inner: vec![],
98 | })?
99 | } else {
100 | crate::store::ZERO_CONTEXT
101 | };
102 |
103 | let frame = Frame::builder(topic, context_id)
104 | .maybe_meta(meta.map(|v| value_to_json(&v)))
105 | .maybe_hash(hash)
106 | .maybe_ttl(ttl)
107 | .build();
108 |
109 | self.output.lock().unwrap().push(frame);
110 |
111 | Ok(PipelineData::Empty)
112 | }
113 | }
114 |
--------------------------------------------------------------------------------
/src/nu/commands/cas_command.rs:
--------------------------------------------------------------------------------
1 | use std::io::Read;
2 |
3 | use nu_engine::CallExt;
4 | use nu_protocol::engine::{Call, Command, EngineState, Stack};
5 | use nu_protocol::{Category, PipelineData, ShellError, Signature, SyntaxShape, Type, Value};
6 |
7 | use crate::store::Store;
8 |
9 | #[derive(Clone)]
10 | pub struct CasCommand {
11 | store: Store,
12 | }
13 |
14 | impl CasCommand {
15 | pub fn new(store: Store) -> Self {
16 | Self { store }
17 | }
18 | }
19 |
20 | impl Command for CasCommand {
21 | fn name(&self) -> &str {
22 | ".cas"
23 | }
24 |
25 | fn signature(&self) -> Signature {
26 | Signature::build(".cas")
27 | .input_output_types(vec![(Type::Nothing, Type::String)])
28 | .required(
29 | "hash",
30 | SyntaxShape::String,
31 | "hash of the content to retrieve",
32 | )
33 | .category(Category::Experimental)
34 | }
35 |
36 | fn description(&self) -> &str {
37 | "Retrieve content from the CAS for the given hash"
38 | }
39 |
40 | fn run(
41 | &self,
42 | engine_state: &EngineState,
43 | stack: &mut Stack,
44 | call: &Call,
45 | _input: PipelineData,
46 | ) -> Result {
47 | let span = call.head;
48 | let hash: String = call.req(engine_state, stack, 0)?;
49 | let hash: ssri::Integrity = hash.parse().map_err(|e| ShellError::GenericError {
50 | error: "I/O Error".into(),
51 | msg: format!("Malformed ssri::Integrity:: {}", e),
52 | span: Some(span),
53 | help: None,
54 | inner: vec![],
55 | })?;
56 |
57 | let mut reader =
58 | self.store
59 | .cas_reader_sync(hash)
60 | .map_err(|e| ShellError::GenericError {
61 | error: "I/O Error".into(),
62 | msg: e.to_string(),
63 | span: Some(span),
64 | help: None,
65 | inner: vec![],
66 | })?;
67 |
68 | let mut contents = Vec::new();
69 | reader
70 | .read_to_end(&mut contents)
71 | .map_err(|e| ShellError::GenericError {
72 | error: "I/O Error".into(),
73 | msg: e.to_string(),
74 | span: Some(span),
75 | help: None,
76 | inner: vec![],
77 | })?;
78 |
79 | // Try to convert to string if valid UTF-8, otherwise return as binary
80 | let value = match String::from_utf8(contents.clone()) {
81 | Ok(string) => Value::String {
82 | val: string,
83 | internal_span: span,
84 | },
85 | Err(_) => Value::Binary {
86 | val: contents,
87 | internal_span: span,
88 | },
89 | };
90 |
91 | Ok(PipelineData::Value(value, None))
92 | }
93 | }
94 |
--------------------------------------------------------------------------------
/src/nu/commands/cat_command.rs:
--------------------------------------------------------------------------------
1 | use nu_engine::CallExt;
2 | use nu_protocol::engine::{Call, Command, EngineState, Stack};
3 | use nu_protocol::{Category, PipelineData, ShellError, Signature, SyntaxShape, Type};
4 |
5 | use crate::store::Store;
6 |
7 | #[derive(Clone)]
8 | pub struct CatCommand {
9 | store: Store,
10 | context_id: scru128::Scru128Id,
11 | }
12 |
13 | impl CatCommand {
14 | pub fn new(store: Store, context_id: scru128::Scru128Id) -> Self {
15 | Self { store, context_id }
16 | }
17 | }
18 |
19 | impl Command for CatCommand {
20 | fn name(&self) -> &str {
21 | ".cat"
22 | }
23 |
24 | fn signature(&self) -> Signature {
25 | Signature::build(".cat")
26 | .input_output_types(vec![(Type::Nothing, Type::Any)])
27 | .named(
28 | "limit",
29 | SyntaxShape::Int,
30 | "limit the number of frames to retrieve",
31 | None,
32 | )
33 | .named(
34 | "last-id",
35 | SyntaxShape::String,
36 | "start from a specific frame ID",
37 | None,
38 | )
39 | .category(Category::Experimental)
40 | }
41 |
42 | fn description(&self) -> &str {
43 | "Reads the event stream and returns frames"
44 | }
45 |
46 | fn run(
47 | &self,
48 | engine_state: &EngineState,
49 | stack: &mut Stack,
50 | call: &Call,
51 | _input: PipelineData,
52 | ) -> Result {
53 | let limit: Option = call.get_flag(engine_state, stack, "limit")?;
54 |
55 | let last_id: Option = call.get_flag(engine_state, stack, "last-id")?;
56 | let last_id: Option = last_id
57 | .as_deref()
58 | .map(|s| s.parse().expect("Failed to parse Scru128Id"));
59 |
60 | let frames = self
61 | .store
62 | .read_sync(last_id.as_ref(), limit, Some(self.context_id))
63 | .collect::>();
64 |
65 | use nu_protocol::Value;
66 |
67 | let output = Value::list(
68 | frames
69 | .into_iter()
70 | .map(|frame| crate::nu::util::frame_to_value(&frame, call.head))
71 | .collect(),
72 | call.head,
73 | );
74 |
75 | Ok(PipelineData::Value(output, None))
76 | }
77 | }
78 |
--------------------------------------------------------------------------------
/src/nu/commands/get_command.rs:
--------------------------------------------------------------------------------
1 | use nu_engine::CallExt;
2 | use nu_protocol::engine::{Call, Command, EngineState, Stack};
3 | use nu_protocol::{Category, PipelineData, ShellError, Signature, SyntaxShape, Type};
4 |
5 | use crate::nu::util;
6 | use crate::store::Store;
7 |
8 | #[derive(Clone)]
9 | pub struct GetCommand {
10 | store: Store,
11 | }
12 |
13 | impl GetCommand {
14 | pub fn new(store: Store) -> Self {
15 | Self { store }
16 | }
17 | }
18 |
19 | impl Command for GetCommand {
20 | fn name(&self) -> &str {
21 | ".get"
22 | }
23 |
24 | fn signature(&self) -> Signature {
25 | Signature::build(".get")
26 | .input_output_types(vec![(Type::Nothing, Type::Any)])
27 | .required("id", SyntaxShape::String, "The ID of the frame to retrieve")
28 | .category(Category::Experimental)
29 | }
30 |
31 | fn description(&self) -> &str {
32 | "Retrieves a frame by its ID from the store"
33 | }
34 |
35 | fn run(
36 | &self,
37 | engine_state: &EngineState,
38 | stack: &mut Stack,
39 | call: &Call,
40 | _input: PipelineData,
41 | ) -> Result {
42 | let id_str: String = call.req(engine_state, stack, 0)?;
43 | let id = id_str.parse().map_err(|e| ShellError::TypeMismatch {
44 | err_message: format!("Invalid ID format: {}", e),
45 | span: call.span(),
46 | })?;
47 |
48 | let store = self.store.clone();
49 |
50 | if let Some(frame) = store.get(&id) {
51 | Ok(PipelineData::Value(
52 | util::frame_to_value(&frame, call.head),
53 | None,
54 | ))
55 | } else {
56 | Err(ShellError::GenericError {
57 | error: "Frame not found".into(),
58 | msg: format!("No frame found with ID: {}", id_str),
59 | span: Some(call.head),
60 | help: None,
61 | inner: vec![],
62 | })
63 | }
64 | }
65 | }
66 |
--------------------------------------------------------------------------------
/src/nu/commands/head_command.rs:
--------------------------------------------------------------------------------
1 | use nu_engine::CallExt;
2 | use nu_protocol::engine::{Call, Command, EngineState, Stack};
3 | use nu_protocol::{Category, PipelineData, ShellError, Signature, SyntaxShape, Type};
4 |
5 | use crate::nu::util;
6 | use crate::store::Store;
7 |
8 | #[derive(Clone)]
9 | pub struct HeadCommand {
10 | store: Store,
11 | context_id: scru128::Scru128Id,
12 | }
13 |
14 | impl HeadCommand {
15 | pub fn new(store: Store, context_id: scru128::Scru128Id) -> Self {
16 | Self { store, context_id }
17 | }
18 | }
19 |
20 | impl Command for HeadCommand {
21 | fn name(&self) -> &str {
22 | ".head"
23 | }
24 |
25 | fn signature(&self) -> Signature {
26 | Signature::build(".head")
27 | .input_output_types(vec![(Type::Nothing, Type::Any)])
28 | .required("topic", SyntaxShape::String, "topic to get head frame from")
29 | .named(
30 | "context",
31 | SyntaxShape::String,
32 | "context ID (defaults to system context)",
33 | None,
34 | )
35 | .category(Category::Experimental)
36 | }
37 |
38 | fn description(&self) -> &str {
39 | "get the most recent frame for a topic"
40 | }
41 |
42 | fn run(
43 | &self,
44 | engine_state: &EngineState,
45 | stack: &mut Stack,
46 | call: &Call,
47 | _input: PipelineData,
48 | ) -> Result {
49 | let topic: String = call.req(engine_state, stack, 0)?;
50 | let context_str: Option = call.get_flag(engine_state, stack, "context")?;
51 | let context_id = if let Some(ctx) = context_str {
52 | ctx.parse::()
53 | .map_err(|e| ShellError::GenericError {
54 | error: "Invalid context ID".into(),
55 | msg: e.to_string(),
56 | span: Some(call.head),
57 | help: None,
58 | inner: vec![],
59 | })?
60 | } else {
61 | self.context_id
62 | };
63 | let span = call.head;
64 |
65 | if let Some(frame) = self.store.head(&topic, context_id) {
66 | Ok(PipelineData::Value(
67 | util::frame_to_value(&frame, span),
68 | None,
69 | ))
70 | } else {
71 | Ok(PipelineData::Empty)
72 | }
73 | }
74 | }
75 |
--------------------------------------------------------------------------------
/src/nu/commands/mod.rs:
--------------------------------------------------------------------------------
1 | pub mod append_command;
2 | pub mod append_command_buffered;
3 | pub mod cas_command;
4 | pub mod cat_command;
5 | pub mod get_command;
6 | pub mod head_command;
7 | pub mod remove_command;
8 |
--------------------------------------------------------------------------------
/src/nu/commands/remove_command.rs:
--------------------------------------------------------------------------------
1 | use std::str::FromStr;
2 |
3 | use nu_engine::CallExt;
4 | use nu_protocol::engine::{Call, Command, EngineState, Stack};
5 | use nu_protocol::{Category, PipelineData, ShellError, Signature, SyntaxShape, Type};
6 |
7 | use scru128::Scru128Id;
8 |
9 | use crate::store::Store;
10 |
11 | #[derive(Clone)]
12 | pub struct RemoveCommand {
13 | store: Store,
14 | }
15 |
16 | impl RemoveCommand {
17 | pub fn new(store: Store) -> Self {
18 | Self { store }
19 | }
20 | }
21 |
22 | impl Command for RemoveCommand {
23 | fn name(&self) -> &str {
24 | ".remove"
25 | }
26 |
27 | fn signature(&self) -> Signature {
28 | Signature::build(".remove")
29 | .input_output_types(vec![(Type::Nothing, Type::Nothing)])
30 | .required("id", SyntaxShape::String, "The ID of the frame to remove")
31 | .category(Category::Experimental)
32 | }
33 |
34 | fn description(&self) -> &str {
35 | "Removes a frame from the store by its ID"
36 | }
37 |
38 | fn run(
39 | &self,
40 | engine_state: &EngineState,
41 | stack: &mut Stack,
42 | call: &Call,
43 | _input: PipelineData,
44 | ) -> Result {
45 | let id_str: String = call.req(engine_state, stack, 0)?;
46 | let id = Scru128Id::from_str(&id_str).map_err(|e| ShellError::TypeMismatch {
47 | err_message: format!("Invalid ID format: {}", e),
48 | span: call.span(),
49 | })?;
50 |
51 | let store = self.store.clone();
52 |
53 | match store.remove(&id) {
54 | Ok(()) => Ok(PipelineData::Empty),
55 | Err(e) => Err(ShellError::GenericError {
56 | error: "Failed to remove frame".into(),
57 | msg: e.to_string(),
58 | span: Some(call.head),
59 | help: None,
60 | inner: vec![],
61 | }),
62 | }
63 | }
64 | }
65 |
--------------------------------------------------------------------------------
/src/nu/mod.rs:
--------------------------------------------------------------------------------
1 | mod config;
2 | mod engine;
3 |
4 | pub mod commands;
5 | pub mod util;
6 | pub use config::{parse_config, parse_config_legacy, CommonOptions, NuScriptConfig, ReturnOptions};
7 | pub use engine::Engine;
8 | pub use util::{frame_to_pipeline, frame_to_value, value_to_json};
9 |
10 | #[cfg(test)]
11 | mod test_commands;
12 | #[cfg(test)]
13 | mod test_engine;
14 |
--------------------------------------------------------------------------------
/src/nu/test_engine.rs:
--------------------------------------------------------------------------------
1 | use nu_protocol::{PipelineData, Span, Value};
2 | use tempfile::TempDir;
3 |
4 | use crate::nu::Engine;
5 | use crate::store::Store;
6 |
7 | fn setup_test_env() -> (Store, Engine) {
8 | let temp_dir = TempDir::new().unwrap();
9 | let store = Store::new(temp_dir.into_path());
10 | let engine = Engine::new().unwrap();
11 | (store, engine)
12 | }
13 |
14 | // Helper to evaluate expressions and get Value results
15 | fn eval_to_value(engine: &Engine, expr: &str) -> Value {
16 | engine
17 | .eval(PipelineData::empty(), expr.to_string())
18 | .unwrap()
19 | .into_value(Span::test_data())
20 | .unwrap()
21 | }
22 |
23 | #[test]
24 | fn test_add_module() {
25 | let (_store, mut engine) = setup_test_env();
26 |
27 | // Add a module that exports two functions
28 | engine
29 | .add_module(
30 | "testmod",
31 | r#"
32 | # Double the input
33 | export def double [x] { $x * 2 }
34 |
35 | # Add then double
36 | export def add_then_double [x, y] {
37 | ($x + $y) * 2
38 | }
39 | "#,
40 | )
41 | .unwrap();
42 |
43 | // Test the double function
44 | let result = eval_to_value(&engine, "testmod double 5");
45 | assert_eq!(result.as_int().unwrap(), 10);
46 |
47 | // Test the add_then_double function
48 | let result = eval_to_value(&engine, "testmod add_then_double 3 4");
49 | assert_eq!(result.as_int().unwrap(), 14);
50 | }
51 |
52 | #[test]
53 | fn test_add_module_syntax_error() {
54 | let (_store, mut engine) = setup_test_env();
55 |
56 | // Try to add a module with invalid syntax
57 | let result = engine.add_module(
58 | "bad_mod",
59 | r#"
60 | export def bad_fn [] {
61 | let x =
62 | }
63 | "#,
64 | );
65 |
66 | assert!(result.is_err());
67 | }
68 |
69 | #[test]
70 | fn test_add_multiple_modules() {
71 | let (_store, mut engine) = setup_test_env();
72 |
73 | // Add first module
74 | engine
75 | .add_module(
76 | "my-math",
77 | r#"
78 | export def add [x, y] { $x + $y }
79 | "#,
80 | )
81 | .unwrap();
82 |
83 | // Add second module
84 | engine
85 | .add_module(
86 | "my-strings",
87 | r#"
88 | export def join [x, y] { $x + $y }
89 | "#,
90 | )
91 | .unwrap();
92 |
93 | // Test both modules work
94 | let num_result = eval_to_value(&engine, "my-math add 5 3");
95 | assert_eq!(num_result.as_int().unwrap(), 8);
96 |
97 | let str_result = eval_to_value(&engine, "my-strings join 'hello ' 'world'");
98 | assert_eq!(str_result.as_str().unwrap(), "hello world");
99 | }
100 |
101 | #[test]
102 | fn test_add_module_env_var_persistence() {
103 | let (_store, mut engine) = setup_test_env();
104 |
105 | // Add a module that sets an environment variable
106 | engine
107 | .add_module("testmod", r#"export-env { $env.MY_VAR = 'hello' }"#)
108 | .unwrap();
109 |
110 | // Verify the environment variable persists
111 | let result = eval_to_value(&engine, "$env.MY_VAR");
112 | assert_eq!(result.as_str().unwrap(), "hello");
113 | }
114 |
115 | #[test]
116 | fn test_engine_env_vars() {
117 | let (_store, engine) = setup_test_env();
118 |
119 | let engine = engine
120 | .with_env_vars([("TEST_VAR".to_string(), "test_value".to_string())])
121 | .unwrap();
122 |
123 | // Test accessing the environment variable
124 | let result = eval_to_value(&engine, "$env.TEST_VAR");
125 | assert_eq!(result.as_str().unwrap(), "test_value");
126 | }
127 |
128 | use nu_engine::eval_block_with_early_return;
129 | use nu_parser::parse;
130 | use nu_protocol::debugger::WithoutDebug;
131 | use nu_protocol::engine::Stack;
132 | use nu_protocol::engine::StateWorkingSet;
133 |
134 | #[test]
135 | fn test_env_var_persistence() {
136 | // this test is just to build understanding of how Nushell works with respect to preserving
137 | // environment variables across evaluations
138 | let (_store, engine) = setup_test_env();
139 | let mut engine = engine;
140 |
141 | // First evaluation - set env var
142 | let mut stack = Stack::new();
143 | let mut working_set = StateWorkingSet::new(&engine.state);
144 | let block = parse(&mut working_set, None, b"$env.TEST_VAR = '123'", false);
145 | let _ = eval_block_with_early_return::(
146 | &engine.state,
147 | &mut stack,
148 | &block,
149 | PipelineData::empty(),
150 | );
151 | engine.state.merge_env(&mut stack).unwrap();
152 |
153 | // Second evaluation - verify env var persists
154 | let result = eval_to_value(&engine, "$env.TEST_VAR");
155 | assert_eq!(result.as_str().unwrap(), "123");
156 | }
157 |
--------------------------------------------------------------------------------
/src/nu/util.rs:
--------------------------------------------------------------------------------
1 | use std::io::Read;
2 | use std::io::Write;
3 |
4 | use nu_protocol::{PipelineData, Record, ShellError, Span, Value};
5 |
6 | use crate::store::Frame;
7 | use crate::store::Store;
8 |
9 | pub fn json_to_value(json: &serde_json::Value, span: Span) -> Value {
10 | match json {
11 | serde_json::Value::Null => Value::nothing(span),
12 | serde_json::Value::Bool(b) => Value::bool(*b, span),
13 | serde_json::Value::Number(n) => {
14 | if let Some(i) = n.as_i64() {
15 | Value::int(i, span)
16 | } else if let Some(f) = n.as_f64() {
17 | Value::float(f, span)
18 | } else {
19 | Value::string(n.to_string(), span)
20 | }
21 | }
22 | serde_json::Value::String(s) => Value::string(s, span),
23 | serde_json::Value::Array(arr) => {
24 | let values: Vec = arr.iter().map(|v| json_to_value(v, span)).collect();
25 | Value::list(values, span)
26 | }
27 | serde_json::Value::Object(obj) => {
28 | let mut record = Record::new();
29 | for (k, v) in obj {
30 | record.push(k, json_to_value(v, span));
31 | }
32 | Value::record(record, span)
33 | }
34 | }
35 | }
36 |
37 | pub fn frame_to_value(frame: &Frame, span: Span) -> Value {
38 | let mut record = Record::new();
39 |
40 | record.push("id", Value::string(frame.id.to_string(), span));
41 | record.push("topic", Value::string(frame.topic.clone(), span));
42 | record.push("context_id", Value::string(frame.context_id, span));
43 |
44 | if let Some(hash) = &frame.hash {
45 | record.push("hash", Value::string(hash.to_string(), span));
46 | }
47 |
48 | if let Some(meta) = &frame.meta {
49 | record.push("meta", json_to_value(meta, span));
50 | }
51 |
52 | Value::record(record, span)
53 | }
54 |
55 | pub fn frame_to_pipeline(frame: &Frame) -> PipelineData {
56 | PipelineData::Value(frame_to_value(frame, Span::unknown()), None)
57 | }
58 |
59 | pub fn value_to_json(value: &Value) -> serde_json::Value {
60 | match value {
61 | Value::Nothing { .. } => serde_json::Value::Null,
62 | Value::Bool { val, .. } => serde_json::Value::Bool(*val),
63 | Value::Int { val, .. } => serde_json::Value::Number((*val).into()),
64 | Value::Float { val, .. } => serde_json::Number::from_f64(*val)
65 | .map(serde_json::Value::Number)
66 | .unwrap_or(serde_json::Value::Null),
67 | Value::String { val, .. } => serde_json::Value::String(val.clone()),
68 | Value::List { vals, .. } => {
69 | serde_json::Value::Array(vals.iter().map(value_to_json).collect())
70 | }
71 | Value::Record { val, .. } => {
72 | let mut map = serde_json::Map::new();
73 | for (k, v) in val.iter() {
74 | map.insert(k.clone(), value_to_json(v));
75 | }
76 | serde_json::Value::Object(map)
77 | }
78 | _ => serde_json::Value::Null,
79 | }
80 | }
81 |
82 | pub fn write_pipeline_to_cas(
83 | input: PipelineData,
84 | store: &Store,
85 | span: Span,
86 | ) -> Result, Box> {
87 | let mut writer = store.cas_writer_sync().map_err(|e| {
88 | Box::new(ShellError::GenericError {
89 | error: "I/O Error".into(),
90 | msg: e.to_string(),
91 | span: Some(span),
92 | help: None,
93 | inner: vec![],
94 | })
95 | })?;
96 |
97 | match input {
98 | PipelineData::Value(value, _) => match value {
99 | Value::Nothing { .. } => Ok(None),
100 | Value::String { val, .. } => {
101 | writer.write_all(val.as_bytes()).map_err(|e| {
102 | Box::new(ShellError::GenericError {
103 | error: "I/O Error".into(),
104 | msg: e.to_string(),
105 | span: Some(span),
106 | help: None,
107 | inner: vec![],
108 | })
109 | })?;
110 |
111 | let hash = writer.commit().map_err(|e| {
112 | Box::new(ShellError::GenericError {
113 | error: "I/O Error".into(),
114 | msg: e.to_string(),
115 | span: Some(span),
116 | help: None,
117 | inner: vec![],
118 | })
119 | })?;
120 |
121 | Ok(Some(hash))
122 | }
123 | Value::Binary { val, .. } => {
124 | writer.write_all(&val).map_err(|e| {
125 | Box::new(ShellError::GenericError {
126 | error: "I/O Error".into(),
127 | msg: e.to_string(),
128 | span: Some(span),
129 | help: None,
130 | inner: vec![],
131 | })
132 | })?;
133 |
134 | let hash = writer.commit().map_err(|e| {
135 | Box::new(ShellError::GenericError {
136 | error: "I/O Error".into(),
137 | msg: e.to_string(),
138 | span: Some(span),
139 | help: None,
140 | inner: vec![],
141 | })
142 | })?;
143 |
144 | Ok(Some(hash))
145 | }
146 | Value::Record { .. } => {
147 | let json = value_to_json(&value);
148 | let json_string = serde_json::to_string(&json).map_err(|e| {
149 | Box::new(ShellError::GenericError {
150 | error: "I/O Error".into(),
151 | msg: e.to_string(),
152 | span: Some(span),
153 | help: None,
154 | inner: vec![],
155 | })
156 | })?;
157 |
158 | writer.write_all(json_string.as_bytes()).map_err(|e| {
159 | Box::new(ShellError::GenericError {
160 | error: "I/O Error".into(),
161 | msg: e.to_string(),
162 | span: Some(span),
163 | help: None,
164 | inner: vec![],
165 | })
166 | })?;
167 |
168 | let hash = writer.commit().map_err(|e| {
169 | Box::new(ShellError::GenericError {
170 | error: "I/O Error".into(),
171 | msg: e.to_string(),
172 | span: Some(span),
173 | help: None,
174 | inner: vec![],
175 | })
176 | })?;
177 |
178 | Ok(Some(hash))
179 | }
180 | _ => Err(Box::new(ShellError::PipelineMismatch {
181 | exp_input_type: format!(
182 | "expected: string, binary, record, or nothing :: received: {:?}",
183 | value.get_type()
184 | ),
185 | dst_span: span,
186 | src_span: value.span(),
187 | })),
188 | },
189 | PipelineData::ListStream(_stream, ..) => {
190 | panic!("ListStream handling is not yet implemented");
191 | }
192 | PipelineData::ByteStream(stream, ..) => {
193 | if let Some(mut reader) = stream.reader() {
194 | let mut buffer = [0; 8192];
195 | loop {
196 | let bytes_read = reader.read(&mut buffer).map_err(|e| {
197 | Box::new(ShellError::GenericError {
198 | error: "I/O Error".into(),
199 | msg: e.to_string(),
200 | span: Some(span),
201 | help: None,
202 | inner: vec![],
203 | })
204 | })?;
205 |
206 | if bytes_read == 0 {
207 | break;
208 | }
209 |
210 | writer.write_all(&buffer[..bytes_read]).map_err(|e| {
211 | Box::new(ShellError::GenericError {
212 | error: "I/O Error".into(),
213 | msg: e.to_string(),
214 | span: Some(span),
215 | help: None,
216 | inner: vec![],
217 | })
218 | })?;
219 | }
220 | }
221 |
222 | let hash = writer.commit().map_err(|e| {
223 | Box::new(ShellError::GenericError {
224 | error: "I/O Error".into(),
225 | msg: e.to_string(),
226 | span: Some(span),
227 | help: None,
228 | inner: vec![],
229 | })
230 | })?;
231 |
232 | Ok(Some(hash))
233 | }
234 | PipelineData::Empty => Ok(None),
235 | }
236 | }
237 |
--------------------------------------------------------------------------------
/src/store/ttl.rs:
--------------------------------------------------------------------------------
1 | use std::collections::HashMap;
2 | use std::time::Duration;
3 |
4 | use serde::{Deserialize, Deserializer, Serialize, Serializer};
5 |
6 | /// Enum representing the TTL (Time-To-Live) for an event.
7 | #[derive(Default, PartialEq, Eq, Clone, Debug)]
8 | pub enum TTL {
9 | #[default]
10 | Forever, // Event is kept indefinitely.
11 | Ephemeral, // Event is not stored; only active subscribers can see it.
12 | Time(Duration), // Event is kept for a custom duration
13 | Head(u32), // Retains only the last n events for a topic (n >= 1).
14 | }
15 |
16 | impl TTL {
17 | /// Converts a `TTL` into its query string representation.
18 | pub fn to_query(&self) -> String {
19 | match self {
20 | TTL::Forever => "ttl=forever".to_string(),
21 | TTL::Ephemeral => "ttl=ephemeral".to_string(),
22 | TTL::Time(duration) => format!("ttl=time:{}", duration.as_millis()),
23 | TTL::Head(n) => format!("ttl=head:{}", n),
24 | }
25 | }
26 |
27 | /// Parses a `TTL` from a query string.
28 | pub fn from_query(query: Option<&str>) -> Result {
29 | // Parse query string into key-value pairs
30 | let params = match query {
31 | None => return Ok(TTL::default()), // Use default TTL if query is None
32 | Some(q) => serde_urlencoded::from_str::>(q)
33 | .map_err(|_| "invalid query string".to_string())?,
34 | };
35 |
36 | // Extract the `ttl` parameter if it exists
37 | if let Some(ttl_str) = params.get("ttl") {
38 | parse_ttl(ttl_str)
39 | } else {
40 | Ok(TTL::default()) // Use default TTL if `ttl` is not present
41 | }
42 | }
43 | }
44 |
45 | impl Serialize for TTL {
46 | fn serialize(&self, serializer: S) -> Result
47 | where
48 | S: Serializer,
49 | {
50 | match self {
51 | TTL::Forever => serializer.serialize_str("forever"),
52 | TTL::Ephemeral => serializer.serialize_str("ephemeral"),
53 | TTL::Time(duration) => {
54 | serializer.serialize_str(&format!("time:{}", duration.as_millis()))
55 | }
56 | TTL::Head(n) => serializer.serialize_str(&format!("head:{}", n)),
57 | }
58 | }
59 | }
60 |
61 | impl<'de> Deserialize<'de> for TTL {
62 | fn deserialize(deserializer: D) -> Result
63 | where
64 | D: Deserializer<'de>,
65 | {
66 | let s: String = Deserialize::deserialize(deserializer)?;
67 | parse_ttl(&s).map_err(serde::de::Error::custom)
68 | }
69 | }
70 |
71 | /// Parses a raw TTL string and converts it to the `TTL` enum.
72 | pub fn parse_ttl(s: &str) -> Result {
73 | match s {
74 | "forever" => Ok(TTL::Forever),
75 | "ephemeral" => Ok(TTL::Ephemeral),
76 | _ if s.starts_with("time:") => {
77 | let duration_str = &s[5..];
78 | let duration = duration_str
79 | .parse::()
80 | .map_err(|_| "Invalid duration for 'time' TTL".to_string())?;
81 | Ok(TTL::Time(Duration::from_millis(duration)))
82 | }
83 | _ if s.starts_with("head:") => {
84 | let n_str = &s[5..];
85 | let n = n_str
86 | .parse::()
87 | .map_err(|_| "Invalid 'n' value for 'head' TTL".to_string())?;
88 | if n < 1 {
89 | Err("'n' must be >= 1 for 'head' TTL".to_string())
90 | } else {
91 | Ok(TTL::Head(n))
92 | }
93 | }
94 | _ => Err("Invalid TTL format".to_string()),
95 | }
96 | }
97 |
--------------------------------------------------------------------------------
/xs.nu:
--------------------------------------------------------------------------------
1 | export alias "h. get" = h. request get
2 | export alias "h. post" = h. request post
3 |
4 | export const XS_CONTEXT_SYSTEM = "0000000000000000000000000"
5 |
6 | def and-then [next: closure --else: closure] {
7 | if ($in | is-not-empty) { do $next } else {
8 | if $else != null { do $else }
9 | }
10 | }
11 |
12 | def or-else [or_else: closure] {
13 | if ($in | is-not-empty) { $in } else { do $or_else }
14 | }
15 |
16 | def conditional-pipe [
17 | condition: bool
18 | action: closure
19 | ] {
20 | if $condition { do $action } else { }
21 | }
22 |
23 | export def xs-addr [] {
24 | $env | get XS_ADDR? | or-else { try { open ~/.config/cross.stream/XS_ADDR | str trim | path expand } } | or-else { "~/.local/share/cross.stream/store" | path expand }
25 | }
26 |
27 | export def xs-context-collect [] {
28 | _cat {context: $XS_CONTEXT_SYSTEM} | reduce --fold {} {|frame, acc|
29 | match $frame.topic {
30 | "xs.context" => ($acc | insert $frame.id $frame.meta?.name?)
31 | "xs.annotate" => (
32 | if $frame.meta?.updates? in $acc {
33 | $acc | update $frame.meta.updates $frame.meta?.name?
34 | } else {
35 | $acc
36 | }
37 | )
38 | _ => $acc
39 | }
40 | } | transpose id name | prepend {
41 | id: $XS_CONTEXT_SYSTEM
42 | name: "system"
43 | }
44 | }
45 |
46 | export def xs-context [selected?: string span?] {
47 | if $selected == null {
48 | return ($env | get XS_CONTEXT?)
49 | }
50 |
51 | xs-context-collect | where id == $selected or name == $selected | try { first | get id } catch {
52 | if $span != null {
53 | error make {
54 | msg: $"context not found: ($selected)"
55 | label: {text: "provided span" span: $span}
56 | }
57 | } else {
58 | error make -u {msg: $"context not found: ($selected)"}
59 | }
60 | }
61 | }
62 |
63 | def _cat [options: record] {
64 | let params = [
65 | (if ($options | get follow? | default false) { "--follow" })
66 | (if ($options | get tail? | default false) { "--tail" })
67 | (if ($options | get all? | default false) { "--all" })
68 |
69 | (if $options.last_id? != null { ["--last-id" $options.last_id] })
70 |
71 | (if $options.limit? != null { ["--limit" $options.limit] })
72 | (if $options.pulse? != null { ["--pulse" $options.pulse] })
73 | (if $options.context? != null { ["--context" $options.context] })
74 | (if $options.topic? != null { ["--topic" $options.topic] })
75 | ] | compact | flatten
76 |
77 | xs cat (xs-addr) ...$params | lines | each {|x| $x | from json }
78 | }
79 |
80 | export def .cat [
81 | --follow (-f) # long poll for new events
82 | --pulse (-p): int # specifies the interval (in milliseconds) to receive a synthetic "xs.pulse" event
83 | --tail (-t) # begin long after the end of the stream
84 | --detail (-d) # include all frame fields in the output
85 | --last-id (-l): string
86 | --limit: int
87 | --context (-c): string # the context to read from
88 | --all (-a) # cat across all contexts
89 | --topic (-T): string # filter by topic
90 | ] {
91 | _cat {
92 | follow: $follow
93 | pulse: $pulse
94 | tail: $tail
95 | last_id: $last_id
96 | limit: $limit
97 | context: (if not $all { (xs-context $context (metadata $context).span) })
98 | all: $all
99 | topic: $topic
100 | } | conditional-pipe (not ($detail or $all)) { each { reject context_id ttl } }
101 | }
102 |
103 | def read_hash [hash?: any] {
104 | match ($hash | describe -d | get type) {
105 | "string" => $hash
106 | "record" => ($hash | get hash?)
107 | _ => null
108 | }
109 | }
110 |
111 | export def .cas [hash?: any] {
112 | let alt = $in
113 | let hash = read_hash (if $hash != null { $hash } else { $alt })
114 | if $hash == null { return }
115 | xs cas (xs-addr) $hash
116 | }
117 |
118 | export def .get [id: string] {
119 | xs get (xs-addr) $id | from json
120 | }
121 |
122 | export def .head [
123 | topic: string
124 | --follow (-f)
125 | --context (-c): string
126 | ] {
127 | let params = [
128 | (xs-context $context (metadata $context).span | and-then { ["--context" $in] })
129 | ] | compact | flatten
130 |
131 | if $follow {
132 | xs head (xs-addr) $topic ...($params) --follow | lines | each {|x| $x | from json }
133 | } else {
134 | xs head (xs-addr) $topic ...($params) | from json
135 | }
136 | }
137 |
138 | # Append an event to the stream
139 | export def .append [
140 | topic: string # The topic to append the event to
141 | --meta: record # Optional metadata to include with the event, provided as a record
142 | --context (-c): string # the context to append to
143 | --ttl: string # Optional Time-To-Live for the event. Supported formats:
144 | # - "forever": The event is kept indefinitely.
145 | # - "ephemeral": The event is not stored; only active subscribers can see it.
146 | # - "time:": The event is kept for a custom duration in milliseconds.
147 | # - "head:": Retains only the last n events for the topic (n must be >= 1).
148 | ] {
149 | xs append (xs-addr) $topic ...(
150 | [
151 | (if $meta != null { ["--meta" ($meta | to json -r)] })
152 | (if $ttl != null { ["--ttl" $ttl] })
153 | (xs-context $context (metadata $context).span | and-then { ["--context" $in] })
154 | ] | compact | flatten
155 | ) | from json
156 | }
157 |
158 | export def .remove [id: string] {
159 | xs remove (xs-addr) $id
160 | }
161 |
162 | export alias .rm = .remove
163 |
164 | export def ".ctx" [
165 | --detail (-d) # return a record with id and name fields
166 | ] {
167 | let id = xs-context | or-else { $XS_CONTEXT_SYSTEM }
168 | let name = xs-context-collect | where id == $id | get name.0
169 | if $detail {
170 | {id: $id} | if $name != null { insert name $name } else { $in }
171 | } else {
172 | $name | default $id
173 | }
174 | }
175 |
176 | export def ".ctx list" [] {
177 | let active = .ctx -d | get id
178 | xs-context-collect | insert active {
179 | $in.id == $active
180 | }
181 | }
182 |
183 | export alias ".ctx ls" = .ctx list
184 |
185 | export def --env ".ctx switch" [id?: string] {
186 | $env.XS_CONTEXT = $id | or-else { .ctx select }
187 | .ctx --detail | get id
188 | }
189 |
190 | export def --env ".ctx new" [name: string] {
191 | .append "xs.context" -c $XS_CONTEXT_SYSTEM --meta {name: $name} | .ctx switch $in.id
192 | }
193 |
194 | export def --env ".ctx rename" [id: string name: string] {
195 | .append "xs.annotate" -c $XS_CONTEXT_SYSTEM --meta {
196 | updates: (xs-context $id (metadata $id).span)
197 | name: $name
198 | }
199 | }
200 |
201 | export def --env ".ctx select" [] {
202 | .ctx list | input list | get id
203 | }
204 |
205 | export def .export [path: string] {
206 | if ($path | path exists) {
207 | print "path exists"
208 | return
209 | }
210 | mkdir ($path | path join "cas")
211 |
212 | xs cat (xs-addr) | save ($path | path join "frames.jsonl")
213 |
214 | open ($path | path join "frames.jsonl") | lines | each { from json | get hash } | uniq | each {|hash|
215 | let hash_64 = $hash | encode base64
216 | let out_path = $"($path)/cas/($hash_64)"
217 | print $out_path
218 | .cas $hash | save $out_path
219 | }
220 | }
221 |
222 | export def .import [path: string] {
223 | glob ([$path "cas"] | path join "*") | each {|x|
224 | let want = ($x | path basename | decode base64 | decode)
225 | let got = cat $x | xs cas-post (xs-addr)
226 | if $got != $want {
227 | return (
228 | error make {
229 | msg: $"hash mismatch got=($got) want=($want)"
230 | }
231 | )
232 | }
233 | $got
234 | }
235 |
236 | open ($path | path join "frames.jsonl") | lines | each {
237 | from json | default "0000000000000000000000000" context_id | to json -r | xs import (xs-addr)
238 | }
239 | }
240 |
--------------------------------------------------------------------------------