├── resources ├── .keep ├── flow.png ├── simulflow.png ├── silero_vad.onnx ├── test-voice.wav ├── silero_vad_half.onnx ├── silero_vad_16k_op15.onnx └── clj-kondo.exports │ └── com.shipclojure │ └── simulflow │ ├── config.edn │ └── hooks │ ├── defframe.clj │ └── defalias.clj ├── doc ├── implementation │ ├── decisions.org │ ├── README.org │ └── drafts.org └── intro.md ├── bin ├── kaocha └── chat ├── dev ├── user.clj ├── malli.clj └── simulflow │ └── dev.clj ├── .clj-kondo └── config.edn ├── tests.edn ├── .dir-locals.el ├── src └── simulflow │ ├── vad │ ├── factory.clj │ └── core.clj │ ├── secrets.clj │ ├── transport │ ├── protocols.clj │ ├── codecs.clj │ ├── text_in.clj │ └── text_out.clj │ ├── onnx.clj │ ├── async.clj │ ├── command.clj │ ├── filters │ └── mute.clj │ ├── processors │ ├── google.clj │ ├── openai.clj │ ├── activity_monitor.clj │ ├── audio_resampler.clj │ ├── system_frame_router.clj │ ├── groq.clj │ ├── deepgram.clj │ └── elevenlabs.clj │ ├── transport.clj │ ├── utils │ ├── request.clj │ └── openai.clj │ └── scenario_manager.clj ├── cljfmt.edn ├── bb.edn ├── .gitignore ├── examples ├── deps.edn └── src │ └── simulflow_examples │ ├── gemini.clj │ ├── local_w_groq.clj │ ├── scenario_example.clj │ ├── local_with_mute_filter.clj │ ├── scenario_example_system_router_mute_filter.clj │ ├── local_w_interruption_support.clj │ ├── local.clj │ └── text_chat.clj ├── .github └── workflows │ └── tests.yaml ├── test └── simulflow │ ├── utils │ └── core_test.clj │ └── util │ └── core_test.clj ├── pom.xml ├── deps.edn ├── TODO.org_archive └── LICENSE /resources/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /doc/implementation/decisions.org: -------------------------------------------------------------------------------- 1 | #+title: Decisions 2 | -------------------------------------------------------------------------------- /bin/kaocha: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | clojure -M:test "$@" 3 | -------------------------------------------------------------------------------- /dev/user.clj: -------------------------------------------------------------------------------- 1 | (ns user 2 | (:require 3 | [simulflow.dev])) 4 | -------------------------------------------------------------------------------- /resources/flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shipclojure/simulflow/HEAD/resources/flow.png -------------------------------------------------------------------------------- /.clj-kondo/config.edn: -------------------------------------------------------------------------------- 1 | {:config-paths ["../resources/clj-kondo.exports/com.shipclojure/simulflow"]} 2 | -------------------------------------------------------------------------------- /resources/simulflow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shipclojure/simulflow/HEAD/resources/simulflow.png -------------------------------------------------------------------------------- /resources/silero_vad.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shipclojure/simulflow/HEAD/resources/silero_vad.onnx -------------------------------------------------------------------------------- /resources/test-voice.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shipclojure/simulflow/HEAD/resources/test-voice.wav -------------------------------------------------------------------------------- /resources/silero_vad_half.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shipclojure/simulflow/HEAD/resources/silero_vad_half.onnx -------------------------------------------------------------------------------- /resources/silero_vad_16k_op15.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shipclojure/simulflow/HEAD/resources/silero_vad_16k_op15.onnx -------------------------------------------------------------------------------- /doc/intro.md: -------------------------------------------------------------------------------- 1 | # Introduction to simulflow 2 | 3 | TODO: write [great documentation](http://jacobian.org/writing/what-to-write/) 4 | -------------------------------------------------------------------------------- /tests.edn: -------------------------------------------------------------------------------- 1 | #kaocha/v1 2 | {:plugins [:noyoda.plugin/swap-actual-and-expected] 3 | :tests [{:id :unit 4 | :source-paths ["src"] 5 | :test-paths ["test"] 6 | :ns-patterns ["-test$"]}]} 7 | -------------------------------------------------------------------------------- /doc/implementation/README.org: -------------------------------------------------------------------------------- 1 | #+title: This directory contains implementation docs & decisions 2 | 3 | Follow this document if you want to understand why things are the way they are 4 | and how it got to this point. 5 | -------------------------------------------------------------------------------- /.dir-locals.el: -------------------------------------------------------------------------------- 1 | ((nil . ((cider-preferred-build-tool . clojure-cli) 2 | (cider-clojure-cli-aliases . ":dev:test:with-examples:clj-reload") 3 | (cider-clojure-cli-parameters . "--port 8000")))) 4 | -------------------------------------------------------------------------------- /src/simulflow/vad/factory.clj: -------------------------------------------------------------------------------- 1 | (ns simulflow.vad.factory 2 | "Standard vad processors supported by simulflow" 3 | (:require 4 | [simulflow.vad.silero :as silero])) 5 | 6 | (def factory 7 | {:vad.analyser/silero silero/create-silero-vad}) 8 | -------------------------------------------------------------------------------- /resources/clj-kondo.exports/com.shipclojure/simulflow/config.edn: -------------------------------------------------------------------------------- 1 | {:lint-as {uncomplicate.commons.core/with-release clojure.core/let 2 | simulflow.async/vthread-loop clojure.core/loop} 3 | :hooks {:analyze-call {simulflow.frame/defframe hooks.defframe/defframe}}} 4 | -------------------------------------------------------------------------------- /src/simulflow/secrets.clj: -------------------------------------------------------------------------------- 1 | (ns simulflow.secrets 2 | (:require 3 | [clojure.edn :as edn] 4 | [clojure.java.io :as io])) 5 | 6 | (defn- secret-map 7 | [] 8 | (edn/read-string (slurp (io/resource "secrets.edn")))) 9 | 10 | (defn secret 11 | [path] 12 | (get-in (secret-map) path)) 13 | -------------------------------------------------------------------------------- /src/simulflow/transport/protocols.clj: -------------------------------------------------------------------------------- 1 | (ns simulflow.transport.protocols) 2 | 3 | (defprotocol FrameSerializer 4 | (serialize-frame [this frame] "Encode a frame to a specific format")) 5 | 6 | (defprotocol FrameDeserializer 7 | (deserialize-frame [this raw-data] "Decode some raw data into a simulflow frame")) 8 | -------------------------------------------------------------------------------- /cljfmt.edn: -------------------------------------------------------------------------------- 1 | ;; `cljfmt` is not used directly with simulflow. I recommend you use 2 | ;; `standard-clojure`: https://github.com/oakmac/standard-clojure-style-js 3 | 4 | ;; This config is here because AI tools like `clojure-mcp` format code so it is 5 | ;; good for the AI formatting to be as compliant as possible with our standard 6 | ;; formatting. 7 | {:indents {#re ".*" [[:inner 0]]} 8 | :remove-surrounding-whitespace? false 9 | :remove-trailing-whitespace? false 10 | :remove-consecutive-blank-lines? false} 11 | -------------------------------------------------------------------------------- /dev/malli.clj: -------------------------------------------------------------------------------- 1 | (ns malli 2 | (:require 3 | [clojure.java.io :as io] 4 | [malli.dev :as malli-dev])) 5 | 6 | (defn export-types [] 7 | ;; collect schemas and start instrumentation 8 | (malli-dev/start!) 9 | 10 | ;; create export file 11 | (let [export-file (io/file "resources/clj-kondo.exports/com.shipclojure/simulflow-types/config.edn")] 12 | 13 | ;; make parents if not exist 14 | (io/make-parents export-file) 15 | 16 | ;; copy the configs 17 | (io/copy 18 | (io/file ".clj-kondo/metosin/malli-types-clj/config.edn") 19 | export-file)) 20 | 21 | ;; clear the cache and stop instrumentation 22 | (malli-dev/stop!)) 23 | 24 | (comment 25 | 26 | (export-types) 27 | 28 | ,) 29 | -------------------------------------------------------------------------------- /src/simulflow/onnx.clj: -------------------------------------------------------------------------------- 1 | (ns simulflow.onnx 2 | "Namespace with various utils for the Onnx framework " 3 | (:import 4 | (ai.onnxruntime OrtEnvironment OrtSession) 5 | (java.util Map))) 6 | 7 | (defn inspect-model [^String model-path] 8 | (let [env (OrtEnvironment/getEnvironment) 9 | session ^OrtSession (.createSession env model-path)] 10 | (println "Input names and shapes:") 11 | (doseq [input ^Map (.getInputInfo session)] 12 | (println " " (.getKey input) "->" (.getValue input))) 13 | (println "Output names and shapes:") 14 | (doseq [output (.getOutputInfo session)] 15 | (println " " (.getKey output) "->" (.getValue output))) 16 | (.close session))) 17 | 18 | (comment 19 | (inspect-model "resources/silero_vad.onnx")) 20 | -------------------------------------------------------------------------------- /bb.edn: -------------------------------------------------------------------------------- 1 | {:min-bb-version "1.2.174" 2 | :paths ["src" "test"] 3 | :deps 4 | {current/deps {:local/root "."}} 5 | 6 | :tasks 7 | {test {:doc "Run all tests" 8 | :task (shell "bin/kaocha")} 9 | jar {:doc "Build jar with latest version" 10 | :task (clojure "-T:build jar")} 11 | pom {:doc "Build pom.xml based on deps.edn" 12 | :task (clojure "-X:deps mvn-pom")} 13 | deploy {:doc "Deploy the latest release" 14 | :task (clojure "-X:deploy")} 15 | format:check {:doc "Check code formatting with standard-clj" 16 | :task (shell "standard-clj" "check" "src" "test" "examples/src")} 17 | format:project {:doc "Run standard-clojure formatting on the entire codebase" 18 | :task (shell "standard-clj" "fix" "src" "test" "examples/src")}}} 19 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .calva/output-window/ 2 | .classpath 3 | .cpcache 4 | .DS_Store 5 | .eastwood 6 | .factorypath 7 | .hg/ 8 | .hgignore 9 | .java-version 10 | .lein-* 11 | .lsp/.cache 12 | .lsp/sqlite.db 13 | .nrepl-history 14 | .nrepl-port 15 | .portal 16 | .project 17 | .rebel_readline_history 18 | .settings 19 | .socket-repl-port 20 | .sw* 21 | .vscode 22 | *.class 23 | *.jar 24 | *.swp 25 | *~ 26 | /checkouts 27 | /classes 28 | /target 29 | /resources/secrets.edn 30 | /core/resources/secrets.edn 31 | /core/.lsp/ 32 | /examples/.calva/ 33 | .idea/ 34 | *.iml 35 | 36 | 37 | # From https://github.com/metabase/metabase/blob/6127271a8964c37a2bbef358ad60eaeb5b3fccb3/.gitignore#L95 38 | **/.clj-kondo/.cache 39 | 40 | # clj-kondo: ignore all except our defined config 41 | .clj-kondo/* 42 | !.clj-kondo/README.md 43 | !.clj-kondo/config.edn 44 | !.clj-kondo/config/ 45 | !.clj-kondo/src/ 46 | !.clj-kondo/test/ 47 | !.clj-kondo/macros/ 48 | /.mcp.json 49 | /.clojure-mcp/ 50 | 51 | # Local documentation files - preserved locally but not tracked 52 | PROJECT_SUMMARY.md 53 | CLAUDE.md 54 | LLM_CODE_STYLE.md 55 | -------------------------------------------------------------------------------- /examples/deps.edn: -------------------------------------------------------------------------------- 1 | {:paths ["src"] 2 | :deps {org.clojure/clojure {:mvn/version "1.12.0"} 3 | org.clojure/data.xml {:mvn/version "0.0.8"} 4 | ring/ring-jetty-adapter {:mvn/version "1.13.0"} 5 | com.shipclojure/simulflow {:local/root "../"} 6 | metosin/reitit {:mvn/version "0.7.2"} 7 | hato/hato {:mvn/version "1.0.0"}} 8 | :aliases {:cider-clj {:extra-deps {cider/cider-nrepl {:mvn/version "0.51.1"}} 9 | :main-opts ["-m" "nrepl.cmdline" "--middleware" "[cider.nrepl/cider-middleware]"]} 10 | :dev {:extra-deps {djblue/portal {:mvn/version "0.58.5"} 11 | criterium/criterium {:mvn/version "0.4.6"} 12 | clj-kondo/clj-kondo {:mvn/version "2024.11.14"}}} 13 | :test {:extra-paths ["test"] 14 | :extra-deps {org.clojure/test.check {:mvn/version "1.1.1"} 15 | midje/midje {:mvn/version "1.10.10"} 16 | io.github.cognitect-labs/test-runner 17 | {:git/tag "v0.5.1" :git/sha "dfb30dd"}}}}} 18 | -------------------------------------------------------------------------------- /.github/workflows/tests.yaml: -------------------------------------------------------------------------------- 1 | # https://practical.li/clojure/continuous-integration/github-actions/ 2 | name: Tests build 3 | on: [push, pull_request] 4 | jobs: 5 | clojure: 6 | runs-on: ubuntu-latest 7 | 8 | steps: 9 | - name: Checkout 10 | uses: actions/checkout@v2 11 | 12 | - name: Prepare java 13 | uses: actions/setup-java@v2 14 | with: 15 | distribution: 'temurin' 16 | java-version: '21' 17 | 18 | - name: Install clojure tools 19 | uses: DeLaGuardo/setup-clojure@4.0 20 | with: 21 | cli: 'latest' # Clojure CLI based on tools.deps 22 | 23 | - name: Cache clojure dependencies 24 | uses: actions/cache@v3 25 | with: 26 | path: | 27 | ~/.m2/repository 28 | ~/.gitlibs 29 | ~/.deps.clj 30 | # List all files containing dependencies: 31 | key: cljdeps-${{ hashFiles('core/deps.edn') }} 32 | # key: cljdeps-${{ hashFiles('deps.edn', 'bb.edn') }} 33 | # key: cljdeps-${{ hashFiles('project.clj') }} 34 | # key: cljdeps-${{ hashFiles('build.boot') }} 35 | restore-keys: cljdeps- 36 | 37 | - name: Run Clj Unit tests 38 | run: bin/kaocha 39 | -------------------------------------------------------------------------------- /src/simulflow/transport/codecs.clj: -------------------------------------------------------------------------------- 1 | (ns simulflow.transport.codecs 2 | (:require 3 | [simulflow.frame :as frame] 4 | [simulflow.transport.protocols :as p] 5 | [simulflow.utils.audio :as audio] 6 | [simulflow.utils.core :as u])) 7 | 8 | (defn deserialize-twilio-data 9 | "Convert twilio message to pipeline frame" 10 | [json-data] 11 | (case (:event json-data) 12 | ;; TODO more cases 13 | "media" (frame/audio-input-raw (-> json-data 14 | :media 15 | :payload 16 | u/decode-base64 17 | audio/ulaw8k->pcm16k)) 18 | nil)) 19 | 20 | (defn make-twilio-serializer [stream-sid & {:keys [convert-audio?] 21 | :or {convert-audio? false}}] 22 | (reify 23 | p/FrameSerializer 24 | (serialize-frame [_ frame] 25 | ;; Convert pipeline frame to Twilio-specific format 26 | (if (frame/audio-output-raw? frame) 27 | (let [{:keys [sample-rate audio]} (:frame/data frame)] 28 | (u/json-str {:event "media" 29 | :streamSid stream-sid 30 | :media {:payload (u/encode-base64 (if convert-audio? 31 | (audio/pcm->ulaw8k audio sample-rate) 32 | audio))}})) 33 | frame)))) 34 | -------------------------------------------------------------------------------- /examples/src/simulflow_examples/gemini.clj: -------------------------------------------------------------------------------- 1 | (ns simulflow-examples.gemini 2 | (:require 3 | [clojure.core.async :as a] 4 | [clojure.core.async.flow :as flow] 5 | [simulflow-examples.local :as local] 6 | [simulflow.async :refer [vthread-loop]] 7 | [simulflow.processors.google :as google] 8 | [simulflow.secrets :refer [secret]] 9 | [simulflow.vad.silero :as silero] 10 | [taoensso.telemere :as t])) 11 | 12 | (comment 13 | (def local-flow-gemini 14 | (local/make-local-flow {:extra-procs {:llm {:proc google/google-llm-process 15 | :args {:llm/model :gemini-2.0-flash 16 | :google/api-key (secret [:google :api-key])}}} 17 | :vad-analyser (silero/create-silero-vad)})) 18 | 19 | (defonce flow-started? (atom false)) 20 | 21 | ;; Start local ai flow - starts paused 22 | (let [{:keys [report-chan error-chan]} (flow/start local-flow-gemini)] 23 | (reset! flow-started? true) 24 | ;; Resume local ai -> you can now speak with the AI 25 | (flow/resume local-flow-gemini) 26 | (vthread-loop [] 27 | (when @flow-started? 28 | (when-let [[msg c] (a/alts!! [report-chan error-chan])] 29 | (when (map? msg) 30 | (t/log! {:level :debug :id (if (= c error-chan) :error :report)} msg)) 31 | (recur))))) 32 | 33 | ;; Stop the conversation 34 | (do 35 | (flow/stop local-flow-gemini) 36 | (reset! flow-started? false)) 37 | ,) 38 | -------------------------------------------------------------------------------- /src/simulflow/vad/core.clj: -------------------------------------------------------------------------------- 1 | (ns simulflow.vad.core) 2 | 3 | (defprotocol VADAnalyzer 4 | "The standard protocol for Voice Activity Detection Analizers." 5 | (analyze-audio [this audio-buffer] 6 | "Analize audio and give back the vad state associated with the current audio-buffer. 7 | Args: 8 | this - the VADAnalizer reification 9 | audio-buffer - byte array representing 16kHz PCM mono audio 10 | 11 | Returns: 12 | `:vad.state/speaking` `:vad.state/starting` `:vad.state/quiet` `:vad.state/stopping`") 13 | 14 | (voice-confidence [this audio-buffer] 15 | "Calculates voice activity confidence for the given audio buffer. 16 | Args: 17 | this - the VADAnalizer reification 18 | audio-buffer - byte array representing 16kHz PCM mono audio 19 | 20 | Returns: 21 | Voice confidence score between 0.0 and 1.0") 22 | 23 | (cleanup [this] 24 | "Cleans up any resources created either on instantiation or running other 25 | vad analysis functions related to this analyser.")) 26 | 27 | (def default-params 28 | "Default parameters for VAD. 29 | :vad/min-confidence - minimum confidence threshold for voice detection 30 | :vad/min-speech-duration-ms - duration to wait before confirming voice start 31 | :vad/min-silence-duration-ms - duration to wait before confirming voice end" 32 | {:vad/min-confidence 0.7 33 | :vad/min-speech-duration-ms 200 34 | :vad/min-silence-duration-ms 800}) 35 | 36 | (defn transition? 37 | [vad-state] 38 | (or (= vad-state :vad.state/stopping) 39 | (= vad-state :vad.state/starting))) 40 | -------------------------------------------------------------------------------- /test/simulflow/utils/core_test.clj: -------------------------------------------------------------------------------- 1 | (ns simulflow.utils.core-test 2 | (:require 3 | [clojure.test :refer [deftest is testing]] 4 | [simulflow.utils.core :as u])) 5 | 6 | (deftest content-type-test 7 | (testing "extracts Content-Type with proper casing" 8 | (is (= "application/json" 9 | (u/content-type {"Content-Type" "application/json"})))) 10 | (testing "extracts content-type with lowercase" 11 | (is (= "text/html" 12 | (u/content-type {"content-type" "text/html"})))) 13 | (testing "extracts :content-type as keyword" 14 | (is (= "application/xml" 15 | (u/content-type {:content-type "application/xml"})))) 16 | (testing "prioritizes Content-Type over content-type" 17 | (is (= "application/json" 18 | (u/content-type {"Content-Type" "application/json" 19 | "content-type" "text/html"})))) 20 | (testing "prioritizes Content-Type over :content-type" 21 | (is (= "application/json" 22 | (u/content-type {"Content-Type" "application/json" 23 | :content-type "text/html"})))) 24 | (testing "prioritizes content-type over :content-type" 25 | (is (= "text/html" 26 | (u/content-type {"content-type" "text/html" 27 | :content-type "application/xml"})))) 28 | (testing "returns nil for empty headers" 29 | (is (nil? (u/content-type {})))) 30 | (testing "returns nil for nil headers" 31 | (is (nil? (u/content-type nil)))) 32 | (testing "returns nil when no content-type variants found" 33 | (is (nil? (u/content-type {"Authorization" "Bearer token"}))))) 34 | -------------------------------------------------------------------------------- /dev/simulflow/dev.clj: -------------------------------------------------------------------------------- 1 | (ns simulflow.dev 2 | (:require 3 | [clj-reload.core :as reload] 4 | [clojure.java.io :as io] 5 | [malli.dev :as malli-dev] 6 | [taoensso.telemere :as t])) 7 | 8 | (t/set-min-level! :debug) 9 | 10 | (defn enable-reflection-warnings! 11 | "Enable reflection warnings for all loaded namespaces and set as default for new ones." 12 | [] 13 | ;; Set default for new namespaces 14 | (alter-var-root #'*warn-on-reflection* (constantly true)) 15 | 16 | ;; Enable for all currently loaded namespaces 17 | (doseq [ns-sym (all-ns)] 18 | (binding [*ns* ns-sym] 19 | (set! *warn-on-reflection* true))) 20 | 21 | (println "Reflection warnings enabled for all namespaces")) 22 | 23 | ;; Enable reflection warnings by default in dev 24 | (enable-reflection-warnings!) 25 | 26 | (defmacro jit [sym] 27 | `(requiring-resolve '~sym)) 28 | 29 | (defonce initiated-clj-reload? (atom false)) 30 | 31 | (defn reset 32 | [] 33 | (when-not @initiated-clj-reload? 34 | (reload/init {:dirs ["src" "dev" "test" "../examples/src"]}) 35 | (reset! initiated-clj-reload? true)) 36 | (reload/reload)) 37 | 38 | (defn export-types [] 39 | ;; collect schemas and start instrumentation 40 | ((jit malli-dev/start!)) 41 | 42 | ;; create export file 43 | (def export-file 44 | (io/file "resources/clj-kondo.exports/com.shipclojure/simulflow_types/config.edn")) 45 | 46 | ;; make parents if not exist 47 | (io/make-parents export-file) 48 | 49 | ;; copy the configs 50 | (io/copy 51 | (io/file ".clj-kondo/metosin/malli-types-clj/config.edn") 52 | export-file) 53 | 54 | ;; clear the cache and stop instrumentation 55 | ((jit malli-dev/stop!))) 56 | 57 | (comment ;; s-: 58 | 59 | (reset) 60 | ,) 61 | 62 | (comment 63 | (export-types) 64 | ,) 65 | -------------------------------------------------------------------------------- /src/simulflow/async.clj: -------------------------------------------------------------------------------- 1 | (ns simulflow.async 2 | (:require 3 | [clojure.core.async :refer [alts!!]] 4 | [clojure.core.async.flow :as flow]) 5 | (:import 6 | (java.util.concurrent Executors))) 7 | 8 | (def virtual-threads-supported? 9 | (try 10 | (Class/forName "java.lang.Thread$Builder$OfVirtual") 11 | true 12 | (catch ClassNotFoundException _ 13 | false))) 14 | 15 | (defonce ^:private vthread-executor 16 | (when virtual-threads-supported? 17 | (Executors/newVirtualThreadPerTaskExecutor))) 18 | 19 | (defn vfuturize 20 | "Like flow/futurize but uses virtual threads when available (Java 21+), 21 | otherwise falls back to the specified executor type (default :mixed)" 22 | ([f & {:keys [exec] 23 | :or {exec :mixed}}] 24 | (if virtual-threads-supported? 25 | (flow/futurize f :exec vthread-executor) 26 | (flow/futurize f :exec exec)))) 27 | 28 | (defmacro vthread 29 | "Executes body in a virtual thread (when available) or falls back to a regular 30 | thread pool. Returns immediately to the calling thread. 31 | 32 | Similar to core.async/thread but leverages virtual threads on Java 21+. 33 | 34 | Example: 35 | (vthread 36 | (let [result (http/get \"https://example.com\")] 37 | (process-result result)))" 38 | [& body] 39 | `((vfuturize (fn [] ~@body)))) 40 | 41 | (defmacro vthread-loop 42 | "Like (vthread (loop ...)). Executes the body in a virtual thread with a loop. 43 | 44 | Example: 45 | (vthread-loop [count 0] 46 | (when (< count 10) 47 | (process-item count) 48 | (recur (inc count))))" 49 | [bindings & body] 50 | `(vthread (loop ~bindings ~@body))) 51 | 52 | (defn drain-channel! 53 | "Drains all pending messages from a channel without blocking. 54 | Returns the number of messages drained." 55 | [ch] 56 | (loop [count 0] 57 | (let [[msg _port] (alts!! [ch] :default ::none)] 58 | (if (= msg ::none) 59 | count 60 | (recur (inc count)))))) 61 | -------------------------------------------------------------------------------- /resources/clj-kondo.exports/com.shipclojure/simulflow/hooks/defframe.clj: -------------------------------------------------------------------------------- 1 | (ns hooks.defframe 2 | (:require 3 | [clj-kondo.hooks-api :as api])) 4 | 5 | (defn defframe [{:keys [node]}] 6 | (let [[name docstring frame-type] (rest (:children node)) 7 | frame-name (api/token-node (symbol (str name))) 8 | pred-name (api/token-node (symbol (str name "?")))] 9 | {:node (api/list-node 10 | [(api/token-node 'do) 11 | ;; Multi-arity function definition 12 | (api/list-node 13 | [(api/token-node 'defn) frame-name docstring 14 | ;; Zero arity (frame name) 15 | (api/list-node 16 | [(api/vector-node []) 17 | (api/list-node 18 | [(api/token-node 'simulflow.frame/create-frame) 19 | frame-type])]) 20 | ;; Single arity: (frame-name data) 21 | (api/list-node 22 | [(api/vector-node [(api/token-node 'data)]) 23 | (api/list-node 24 | [(api/token-node 'simulflow.frame/create-frame) 25 | frame-type 26 | (api/token-node 'data)])]) 27 | ;; Two arity: (frame-name data opts) 28 | (api/list-node 29 | [(api/vector-node [(api/token-node 'data) (api/token-node 'opts)]) 30 | (api/list-node 31 | [(api/token-node 'simulflow.frame/create-frame) 32 | frame-type 33 | (api/token-node 'data) 34 | (api/token-node 'opts)])])]) 35 | ;; Predicate function 36 | (api/list-node 37 | [(api/token-node 'defn) pred-name (str "Checks if frame is of type " frame-name) 38 | (api/vector-node [(api/token-node 'frame)]) 39 | (api/list-node 40 | [(api/token-node 'instance?) 41 | frame-name 42 | (api/token-node 'frame)])])])})) 43 | -------------------------------------------------------------------------------- /examples/src/simulflow_examples/local_w_groq.clj: -------------------------------------------------------------------------------- 1 | (ns simulflow-examples.local-w-groq 2 | (:require 3 | [clojure.core.async :as a] 4 | [clojure.core.async.flow :as flow] 5 | [simulflow-examples.local :as local] 6 | [simulflow.processors.groq :as groq] 7 | [simulflow.secrets :refer [secret]] 8 | [taoensso.telemere :as t])) 9 | 10 | (comment 11 | 12 | (def local-flow-groq (local/make-local-flow {:extra-procs {:llm {:proc groq/groq-llm-process 13 | :args {:llm/model "llama-3.3-70b-versatile" 14 | :groq/api-key (secret [:groq :api-key])}}} 15 | :llm-context {:messages 16 | [{:role "system" 17 | :content "You are a voice agent operating via phone. Be 18 | concise in your answers. The input you receive comes from a 19 | speech-to-text (transcription) system that isn't always 20 | efficient and may send unclear text. Ask for 21 | clarification when you're unsure what the person said."}] 22 | :tools []}})) 23 | 24 | (defonce flow-started? (atom false)) 25 | 26 | ;; Start local ai flow - starts paused 27 | (let [{:keys [report-chan error-chan]} (flow/start local-flow-groq)] 28 | (reset! flow-started? true) 29 | ;; Resume local ai -> you can now speak with the AI 30 | (flow/resume local-flow-groq) 31 | (a/thread 32 | (loop [] 33 | (when @flow-started? 34 | (when-let [[msg c] (a/alts!! [report-chan error-chan])] 35 | (when (map? msg) 36 | (t/log! {:level :debug :id (if (= c error-chan) :error :report)} msg)) 37 | (recur)))))) 38 | 39 | ;; Stop the conversation 40 | (do 41 | (flow/stop local-flow-groq) 42 | (reset! flow-started? false)) 43 | 44 | ,) 45 | -------------------------------------------------------------------------------- /test/simulflow/util/core_test.clj: -------------------------------------------------------------------------------- 1 | (ns simulflow.util.core-test 2 | (:require 3 | [clojure.test :refer [deftest is testing]] 4 | [simulflow.utils.core :refer [ends-with-sentence?]])) 5 | 6 | (deftest ends-with-sentence?-test 7 | (testing "Basic sentence endings" 8 | (is (ends-with-sentence? "This is a sentence.")) 9 | (is (ends-with-sentence? "Is this a question?")) 10 | (is (ends-with-sentence? "What an exclamation!")) 11 | (is (ends-with-sentence? "First part; second part")) 12 | (is (ends-with-sentence? "One thing: another thing"))) 13 | 14 | (testing "Asian punctuation marks" 15 | (is (ends-with-sentence? "这是一个句子。")) 16 | (is (ends-with-sentence? "这是问题吗？")) 17 | (is (ends-with-sentence? "多么令人兴奋！")) 18 | (is (ends-with-sentence? "第一部分：")) 19 | (is (ends-with-sentence? "第一个；"))) 20 | 21 | (testing "Should not match abbreviations with periods" 22 | (is (not (ends-with-sentence? "U.S.A."))) 23 | (is (ends-with-sentence? "etc."))) 24 | 25 | (testing "Should not match after numbers" 26 | (is (not (ends-with-sentence? "1."))) 27 | (is (not (ends-with-sentence? "Chapter 2."))) 28 | (is (not (ends-with-sentence? "Section 3.2.")))) 29 | 30 | (testing "Should not match time markers" 31 | (is (not (ends-with-sentence? "3 a."))) 32 | (is (not (ends-with-sentence? "11 p."))) 33 | (is (ends-with-sentence? "9 a.m.")) 34 | (is (ends-with-sentence? "10 p.m."))) 35 | 36 | (testing "Should not match after titles" 37 | (is (not (ends-with-sentence? "Mr."))) 38 | (is (not (ends-with-sentence? "Mrs."))) 39 | (is (not (ends-with-sentence? "Ms."))) 40 | (is (not (ends-with-sentence? "Dr."))) 41 | (is (not (ends-with-sentence? "Prof.")))) 42 | 43 | (testing "Complex sentences" 44 | (is (ends-with-sentence? "The U.S.A. is a country.")) 45 | (is (ends-with-sentence? "She has a Ph.D. in biology.")) 46 | (is (ends-with-sentence? "Mr. Smith went home."))) 47 | 48 | (testing "Edge cases" 49 | (is (not (ends-with-sentence? ""))) 50 | (is (not (ends-with-sentence? "Mrs"))) 51 | (is (ends-with-sentence? "Yes!"))) 52 | 53 | (testing "Mixed punctuation" 54 | (is (ends-with-sentence? "Really?!")) 55 | (is (ends-with-sentence? "No way...!")) 56 | (is (ends-with-sentence? "First: then!"))) 57 | 58 | (testing "Whitespace handling" 59 | (is (ends-with-sentence? "End of sentence. ")) 60 | (is (ends-with-sentence? "Question? ")) 61 | (is (not (ends-with-sentence? "Incomplete ")))) 62 | 63 | (testing "Numbers not followed by periods" 64 | (is (ends-with-sentence? "Score: 5")))) 65 | -------------------------------------------------------------------------------- /resources/clj-kondo.exports/com.shipclojure/simulflow/hooks/defalias.clj: -------------------------------------------------------------------------------- 1 | (ns hooks.defalias 2 | (:require 3 | [clj-kondo.hooks-api :as api])) 4 | 5 | (defn defalias [{:keys [node]}] 6 | (let [children (rest (:children node)) 7 | [alias src & _] children] 8 | (cond 9 | ;; Two argument form: (defalias alias-name source-symbol) 10 | (and alias src) 11 | {:node (api/list-node 12 | [(api/token-node 'def) 13 | alias 14 | src])} 15 | 16 | ;; Single argument form: (defalias source-symbol) 17 | alias 18 | (let [src-symbol (:value alias) 19 | alias-name (api/token-node (symbol (name src-symbol)))] 20 | {:node (api/list-node 21 | [(api/token-node 'def) 22 | alias-name 23 | alias])}) 24 | 25 | ;; Fallback 26 | :else 27 | {:node (api/list-node [(api/token-node 'do)])}))) 28 | 29 | (defn defaliases [{:keys [node]}] 30 | (let [children (rest (:children node)) 31 | ;; Generate a def for each alias clause 32 | defs (mapv (fn [clause] 33 | (cond 34 | ;; Simple symbol form: (defaliases some.ns/symbol) 35 | (api/token-node? clause) 36 | (let [src-symbol (:value clause) 37 | alias-name (api/token-node (symbol (name src-symbol)))] 38 | (api/list-node 39 | [(api/token-node 'def) 40 | alias-name 41 | clause])) 42 | 43 | ;; Map form: {:alias my-name :src some.ns/symbol} 44 | (api/map-node? clause) 45 | (let [map-children (:children clause) 46 | ;; Parse key-value pairs 47 | pairs (partition 2 map-children) 48 | alias-pair (first (filter #(= :alias (:value (first %))) pairs)) 49 | src-pair (first (filter #(= :src (:value (first %))) pairs)) 50 | alias-name (when alias-pair (second alias-pair)) 51 | src-name (when src-pair (second src-pair))] 52 | (if (and alias-name src-name) 53 | (api/list-node 54 | [(api/token-node 'def) 55 | alias-name 56 | src-name]) 57 | ;; Fallback if we can't parse the map 58 | (api/list-node [(api/token-node 'do)]))) 59 | 60 | :else 61 | ;; Unknown form, just emit a do 62 | (api/list-node [(api/token-node 'do)]))) 63 | children)] 64 | {:node (api/list-node 65 | (into [(api/token-node 'do)] 66 | defs))})) 67 | -------------------------------------------------------------------------------- /bin/chat: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Simulflow Text Chat Demo 4 | # Interactive text-based chat with AI using simulflow 5 | 6 | set -e 7 | 8 | # Colors for output 9 | RED='\033[0;31m' 10 | GREEN='\033[0;32m' 11 | BLUE='\033[0;34m' 12 | YELLOW='\033[1;33m' 13 | NC='\033[0m' # No Color 14 | 15 | # Function to print colored output 16 | print_color() { 17 | echo -e "${1}${2}${NC}" 18 | } 19 | 20 | # Check if we're in the right directory 21 | if [ ! -f "deps.edn" ]; then 22 | print_color $RED "❌ Error: This script must be run from the simulflow project root directory" 23 | exit 1 24 | fi 25 | 26 | # Check if secrets.edn exists 27 | if [ ! -f "resources/secrets.edn" ]; then 28 | print_color $RED "❌ Error: resources/secrets.edn not found" 29 | print_color $YELLOW "Please create resources/secrets.edn with your OpenAI API key:" 30 | echo '{:openai {:new-api-sk "your-openai-api-key-here"}}' 31 | exit 1 32 | fi 33 | 34 | # Parse command line arguments 35 | ARGS="" 36 | HELP=false 37 | 38 | while [[ $# -gt 0 ]]; do 39 | case $1 in 40 | --debug) 41 | ARGS="$ARGS --debug" 42 | shift 43 | ;; 44 | --scenario) 45 | ARGS="$ARGS --scenario" 46 | shift 47 | ;; 48 | --help|-h) 49 | HELP=true 50 | shift 51 | ;; 52 | *) 53 | print_color $RED "❌ Unknown option: $1" 54 | HELP=true 55 | shift 56 | ;; 57 | esac 58 | done 59 | 60 | # Show help 61 | if [ "$HELP" = true ]; then 62 | echo "Simulflow Text Chat Demo" 63 | echo "" 64 | echo "Usage: $0 [OPTIONS]" 65 | echo "" 66 | echo "Options:" 67 | echo " --debug Enable debug mode (verbose logging)" 68 | echo " --scenario Use a scenario (structured conversation)" 69 | echo " --help, -h Show this help message" 70 | echo "" 71 | echo "Examples:" 72 | echo " $0 # Start normal open-ended chat" 73 | echo " $0 --debug # Start with debug logging" 74 | echo " $0 --scenario # Start with a scenario" 75 | echo " $0 --debug --scenario # Debug mode with scenario" 76 | echo "" 77 | echo "Once running:" 78 | echo " • Type messages and press Enter" 79 | echo " • Input is blocked while assistant responds" 80 | echo " • Ask to 'quit chat' to exit gracefully (normal chat)" 81 | echo " • Scenario will end automatically when complete" 82 | echo " • Use Ctrl+C to force exit" 83 | exit 0 84 | fi 85 | 86 | # Show startup banner 87 | print_color $GREEN "🚀 Starting Simulflow Text Chat Demo..." 88 | print_color $BLUE "📁 Project: $(pwd)" 89 | 90 | if [[ "$ARGS" == *"--debug"* ]]; then 91 | print_color $YELLOW "🐛 Debug mode enabled" 92 | fi 93 | 94 | if [[ "$ARGS" == *"--scenario"* ]]; then 95 | print_color $YELLOW "🎭 Scenario mode enabled" 96 | fi 97 | 98 | print_color $GREEN "🔧 Loading Clojure dependencies..." 99 | 100 | # Run the chat demo 101 | print_color $GREEN "🤖 Launching interactive chat..." 102 | echo "" 103 | 104 | exec clj -M:with-examples -m simulflow-examples.text-chat $ARGS 105 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | com.shipclojure 5 | jar 6 | simulflow 7 | shipclojure/simulflow 8 | A Clojure framework for building real-time voice-enabled AI applications. simulflow handles the orchestration of speech recognition, audio processing, and AI service integration with the elegance of functional programming. 9 | https://github.com/shipclojure/simulflow 10 | 11 | 12 | Eclipse Public License 13 | http://www.eclipse.org/legal/epl-v10.html 14 | 15 | 16 | 17 | 18 | Ovi Stoica 19 | 20 | 21 | 22 | https://github.com/shipclojure/simulflow 23 | scm:git:git://github.com/shipclojure/simulflow.git 24 | scm:git:ssh://git@github.com/shipclojure/simulflow.git 25 | v0.1.8-alpha 26 | 27 | v0.1.8-alpha 28 | 29 | src 30 | 31 | 32 | 33 | clojars 34 | https://repo.clojars.org/ 35 | 36 | 37 | 38 | 39 | clojars 40 | Clojars repository 41 | https://clojars.org/repo 42 | 43 | 44 | 45 | 46 | org.clojure 47 | clojure 48 | 1.12.0 49 | 50 | 51 | hato 52 | hato 53 | 1.1.0-SNAPSHOT 54 | 55 | 56 | com.microsoft.onnxruntime 57 | onnxruntime 58 | 1.22.0 59 | 60 | 61 | org.uncomplicate 62 | clojure-sound 63 | 0.3.0 64 | 65 | 66 | metosin 67 | malli 68 | 0.19.1 69 | 70 | 71 | metosin 72 | jsonista 73 | 0.3.13 74 | 75 | 76 | org.clojure 77 | core.async 78 | 1.9.808-alpha1 79 | 80 | 81 | com.taoensso 82 | telemere 83 | 1.0.1 84 | 85 | 86 | 87 | -------------------------------------------------------------------------------- /src/simulflow/command.clj: -------------------------------------------------------------------------------- 1 | (ns simulflow.command 2 | (:require 3 | [clojure.string :as str] 4 | [hato.client :as http] 5 | [simulflow.utils.core :refer [content-type json-str without-nils]] 6 | [simulflow.utils.request :refer [sse-request]])) 7 | 8 | ;;; Command System Utilities 9 | ;;; These utilities support the Data Centric Async Processor Pattern 10 | ;;; by processing commands generated by transform functions 11 | 12 | (defn make-command 13 | "Create a command data structure for side effects" 14 | ([kind data] 15 | {:command/kind kind 16 | :command/data data}) 17 | ([kind data id] 18 | {:command/kind kind 19 | :command/data data 20 | :command/id id})) 21 | 22 | (defn sse-request-command 23 | "Create an SSE request command for streaming API calls" 24 | ([params] 25 | (make-command :command/sse-request (without-nils params))) 26 | ([url method body & {:keys [headers timeout-ms buffer-size id]}] 27 | (make-command :command/sse-request 28 | (without-nils {:url url 29 | :method method 30 | :body body 31 | :headers headers 32 | :timeout-ms timeout-ms 33 | :buffer-size buffer-size}) 34 | id))) 35 | 36 | (defn http-request-command 37 | "Create an HTTP request command for regular API calls" 38 | [url method body & {:keys [headers timeout-ms id]}] 39 | (make-command :command/http-request 40 | (without-nils {:url url 41 | :method method 42 | :body body 43 | :headers headers 44 | :timeout-ms timeout-ms}) 45 | id)) 46 | 47 | ;;; Command Handlers 48 | ;;; These functions process commands by executing the side effects 49 | 50 | (defmulti handle-command 51 | "Execute a command by dispatching on its kind" 52 | (fn [command] (:command/kind command))) 53 | 54 | (defmethod handle-command :command/sse-request 55 | [command] 56 | (let [{:keys [url method body headers timeout-ms buffer-size]} (:command/data command) 57 | ;; JSON stringify the body if it's a map/collection and content-type is JSON 58 | final-body (if (and (or (map? body) (coll? body)) 59 | (some-> (content-type headers) 60 | (str/includes? "application/json"))) 61 | (json-str body) 62 | body)] 63 | (:body (sse-request {:request (without-nils {:url url 64 | :method method 65 | :body final-body 66 | :headers headers 67 | :timeout-ms timeout-ms}) 68 | :params (without-nils {:buffer-size buffer-size 69 | :stream/close? true})})))) 70 | 71 | (defmethod handle-command :command/http-request 72 | [command] 73 | (let [{:keys [url method body headers timeout-ms]} (:command/data command)] 74 | (http/request (without-nils {:url url 75 | :method method 76 | :body body 77 | :headers headers 78 | :timeout timeout-ms})))) 79 | 80 | (defmethod handle-command :default 81 | [command] 82 | (throw (ex-info "Unknown command kind" {:command command}))) 83 | -------------------------------------------------------------------------------- /src/simulflow/filters/mute.clj: -------------------------------------------------------------------------------- 1 | (ns simulflow.filters.mute 2 | "This filter handles muting the user input based on specific strategies: 3 | - A function call is in progress 4 | - Bot's first speech (introductions) 5 | - Bot speech (don't process user speech while bot is speaking) 6 | " 7 | (:require 8 | [clojure.core.async.flow :as flow] 9 | [clojure.set :as set] 10 | [simulflow.frame :as frame] 11 | [simulflow.schema :as schema])) 12 | 13 | (def MuteStrategy 14 | [:enum 15 | ;; Mute user during introduction 16 | :mute.strategy/first-speech 17 | ;; Mute user during bot tool calls 18 | :mute.strategy/tool-call 19 | ;; Mute user during all bot speech 20 | :mute.strategy/bot-speech]) 21 | 22 | (def MuteFilterConfig 23 | [:map 24 | [:mute/strategies 25 | {:description "Collection of strategies used to trigger muting the user input."} 26 | [:set MuteStrategy]]]) 27 | 28 | (def describe 29 | {:ins {:in "Channel for normal frames" 30 | :sys-in "Channel for system frames"} 31 | :outs {:sys-out "Channel for mute-start/stop frames"} 32 | :params (schema/->describe-parameters MuteFilterConfig)}) 33 | 34 | (defn init 35 | [params] 36 | (let [parsed-params (schema/parse-with-defaults MuteFilterConfig params)] 37 | parsed-params)) 38 | 39 | (defn transform 40 | [{:keys [mute/strategies ::muted?] :as state} _ msg] 41 | (cond 42 | ;; Function call strategy 43 | (and (frame/llm-tool-call-request? msg) 44 | (strategies :mute.strategy/tool-call) 45 | (not muted?)) 46 | [(assoc state ::muted? true) (frame/send (frame/mute-input-start))] 47 | 48 | (and (frame/llm-tool-call-result? msg) 49 | (strategies :mute.strategy/tool-call) 50 | muted?) 51 | [(assoc state ::muted? false) (frame/send (frame/mute-input-stop))] 52 | 53 | ;; bot speech & first-speech strategies 54 | (and (frame/bot-speech-start? msg) 55 | (seq (set/intersection strategies #{:mute.strategy/first-speech :mute.strategy/bot-speech})) 56 | (not muted?)) 57 | (let [emit-mute-first-speech? (and (strategies :mute.strategy/first-speech) 58 | (not (true? (::first-speech-started? state)))) 59 | emit-mute-bot-speech? (strategies :mute.strategy/bot-speech) 60 | emit-mute? (or emit-mute-first-speech? emit-mute-bot-speech?) 61 | 62 | ns (cond-> state 63 | emit-mute-first-speech? (assoc ::first-speech-started? true) 64 | emit-mute? (assoc ::muted? true))] 65 | [ns (when emit-mute? (frame/send (frame/mute-input-start)))]) 66 | 67 | (and (frame/bot-speech-stop? msg) 68 | (seq (set/intersection strategies #{:mute.strategy/first-speech :mute.strategy/bot-speech})) 69 | muted?) 70 | (let [emit-unmute-first-speech? (and (strategies :mute.strategy/first-speech) 71 | (true? (::first-speech-started? state)) 72 | (not (true? (::first-speech-ended? state)))) 73 | emit-unmute-bot-speech? (strategies :mute.strategy/bot-speech) 74 | emit-unmute? (or emit-unmute-first-speech? emit-unmute-bot-speech?) 75 | 76 | ns (cond-> state 77 | emit-unmute-first-speech? (assoc ::first-speech-ended? true) 78 | emit-unmute? (assoc ::muted? false))] 79 | [ns (when emit-unmute? (frame/send (frame/mute-input-stop)))]) 80 | 81 | :else 82 | [state])) 83 | 84 | (defn processor-fn 85 | ([] describe) 86 | ([params] (init params)) 87 | ([state _transition] state) 88 | ([state in msg] 89 | (transform state in msg))) 90 | 91 | (def process (flow/process processor-fn)) 92 | -------------------------------------------------------------------------------- /src/simulflow/transport/text_in.clj: -------------------------------------------------------------------------------- 1 | (ns simulflow.transport.text-in 2 | (:refer-clojure :exclude [send]) 3 | (:require 4 | [clojure.core.async :as a] 5 | [clojure.core.async.flow :as flow] 6 | [clojure.string :as str] 7 | [simulflow.async :refer [vthread-loop]] 8 | [simulflow.frame :as frame] 9 | [taoensso.telemere :as t]) 10 | (:import 11 | (java.io BufferedReader InputStreamReader))) 12 | 13 | (defn text-input-transform 14 | "Transform function handles stdin input and LLM response lifecycle for blocking" 15 | [state input-port frame] 16 | (let [{:keys [blocked?]} state] 17 | (cond 18 | ;; Handle LLM response lifecycle via :sys-in for blocking 19 | (and (= input-port :sys-in) (frame/llm-full-response-start? frame)) 20 | [(assoc state :blocked? true) {}] 21 | (and (= input-port :sys-in) (frame/llm-full-response-end? frame)) 22 | [(assoc state :blocked? false) {}] 23 | ;; Handle text input from stdin - transform decides whether to process 24 | (and (= input-port ::stdin) (string? frame)) 25 | (if blocked? 26 | (do 27 | (t/log! {:level :debug :id :text-input} "Ignoring input while LLM is responding") 28 | [state {}]) 29 | (let [user-text (str/trim frame)] 30 | (if (not-empty user-text) 31 | ;; Emit the standard speech sequence: start -> transcription -> stop 32 | [state (frame/send 33 | (frame/user-speech-start true) 34 | (frame/transcription user-text) 35 | (frame/user-speech-stop true))] 36 | [state {}]))) 37 | :else [state {}]))) 38 | 39 | (defn text-input-init! 40 | "Initialize text input processor with stdin reading loop (no prompt handling)" 41 | [params] 42 | (let [stdin-ch (a/chan 1024) 43 | running? (atom true) 44 | ;; Create buffered reader for stdin 45 | reader (BufferedReader. (InputStreamReader. System/in)) 46 | ;; Close function to be called on stop 47 | close-fn (fn [] 48 | (reset! running? false) 49 | (try (.close reader) (catch Exception _)) 50 | (a/close! stdin-ch))] 51 | ;; Start stdin reading loop in virtual thread (no prompts) 52 | (vthread-loop [] 53 | (when @running? 54 | (try 55 | ;; Block on stdin read - this is safe in virtual threads 56 | (when-let [line (.readLine reader)] 57 | (when @running? 58 | (when-not (a/offer! stdin-ch line) 59 | (t/log! :warn "stdin channel full, dropping input")))) 60 | (catch Exception e 61 | (if @running? 62 | (do 63 | (t/log! {:level :error :id :text-input :error e} "Error reading from stdin") 64 | (Thread/sleep 100)) 65 | ;; Normal shutdown, don't log error 66 | nil))) 67 | (recur))) 68 | ;; Set up state with channels 69 | {::flow/in-ports {::stdin stdin-ch} 70 | ::close close-fn 71 | :blocked? false})) 72 | 73 | (defn text-input-processor 74 | "Multi-arity text input processor following simulflow patterns" 75 | ([] 76 | ;; Describe processor 77 | {:ins {:in "Text input frames" 78 | :sys-in "System frames (LLM response lifecycle)"} 79 | :outs {:out "User speech frames (transcription sequence)" 80 | :sys-out "System frames"} 81 | :params {}}) 82 | ([config] 83 | ;; Init processor 84 | (text-input-init! config)) 85 | ([state transition] 86 | ;; Handle transitions 87 | (case transition 88 | ::flow/stop (when-let [close-fn (::close state)] 89 | (close-fn)) 90 | state)) 91 | ([state input-port frame] 92 | ;; Transform function 93 | (text-input-transform state input-port frame))) 94 | 95 | ;; Export for use in flows 96 | (def text-input-process 97 | (flow/process text-input-processor)) 98 | -------------------------------------------------------------------------------- /deps.edn: -------------------------------------------------------------------------------- 1 | {:paths ["src" "resources"] 2 | :deps {org.clojure/clojure {:mvn/version "1.12.0"} 3 | org.uncomplicate/clojure-sound {:mvn/version "0.3.0"} 4 | com.taoensso/telemere {:mvn/version "1.0.1"} 5 | metosin/malli {:mvn/version "0.19.1"} 6 | org.clojure/core.async {:mvn/version "1.9.808-alpha1"} 7 | metosin/jsonista {:mvn/version "0.3.13"} 8 | com.microsoft.onnxruntime/onnxruntime {:mvn/version "1.22.0"} 9 | hato/hato {:mvn/version "1.1.0-SNAPSHOT"}} 10 | 11 | :mvn/repos {"central" {:url "https://repo1.maven.org/maven2/"} 12 | "clojars" {:url "https://repo.clojars.org/"}} 13 | 14 | :pom {:group-id "com.shipclojure" 15 | :artifact-id "simulflow" 16 | :version "0.1.8-alpha" 17 | :name "simulflow" 18 | :description "A Clojure framework for building real-time voice-enabled AI applications" 19 | :url "https://github.com/shipclojure/simulflow" 20 | :scm {:url "https://github.com/shipclojure/simulflow" 21 | :tag "v0.1.8-alpha" 22 | :connection "scm:git:git://github.com/shipclojure/simulflow.git" 23 | :dev-connection "scm:git:ssh://git@github.com/shipclojure/simulflow.git"} 24 | :licenses [{:name "Eclipse Public License" 25 | :url "http://www.eclipse.org/legal/epl-v10.html"}] 26 | :developers [{:name "Ovi Stoica"}]} 27 | :aliases 28 | {:build {:deps {io.github.clojure/tools.build {:mvn/version "0.9.4"} 29 | slipset/deps-deploy {:mvn/version "0.2.1"}} 30 | :ns-default build} 31 | :run {:main-opts ["-m" "simulflow.transport.local.audio"] 32 | :exec-fn simulflow.transport.local.audio/main} 33 | :dev {:extra-paths ["dev"] 34 | ;; Check schema for each frame instantiation (not recommended in prod 35 | ;; since many schemas are created) 36 | :jvm-opts ["-Dsimulflow.frame.schema-checking=true"] 37 | :extra-deps {djblue/portal {:mvn/version "0.58.5"} 38 | criterium/criterium {:mvn/version "0.4.6"} 39 | clj-kondo/clj-kondo {:mvn/version "2025.06.05"}}} 40 | :cider-clj {:extra-deps {cider/cider-nrepl {:mvn/version "0.52.0"}} 41 | :main-opts ["-m" "nrepl.cmdline" 42 | "--middleware" "[cider.nrepl/cider-middleware]" 43 | "--port" "52158"]} 44 | :test {:extra-paths ["test"] 45 | :main-opts ["-m" "kaocha.runner"] 46 | :extra-deps {org.clojure/tools.namespace {:mvn/version "1.4.4"} 47 | org.clojure/test.check {:mvn/version "1.1.1"} 48 | lambdaisland/kaocha {:mvn/version "1.0.732"} 49 | kaocha-noyoda/kaocha-noyoda {:mvn/version "2019-06-03"}}} 50 | :with-examples {:extra-deps {org.clojure/data.xml {:mvn/version "0.0.8"} 51 | ring/ring-jetty-adapter {:mvn/version "1.14.2"} 52 | ring/ring-core {:mvn/version "1.14.2"} 53 | metosin/reitit {:mvn/version "0.9.1"}} 54 | :extra-paths ["./examples/src"]} 55 | :clj-kondo {:replace-deps {clj-kondo/clj-kondo {:mvn/version "2025.06.05"}} 56 | :main-opts ["-m" "clj-kondo.main"]} 57 | :storm {:classpath-overrides {org.clojure/clojure nil} 58 | :extra-deps {com.github.flow-storm/clojure {:mvn/version "1.12.0-3"} 59 | com.github.flow-storm/flow-storm-dbg {:mvn/version "4.2.0-SNAPSHOT"} 60 | com.github.flow-storm/flow-storm-async-flow-plugin {:mvn/version "1.0.0-SNAPSHOT"}} 61 | :jvm-opts ["-Dclojure.storm.instrumentEnable=true" 62 | "-Dclojure.storm.instrumentOnlyPrefixes=clojure.core.async.flow,simulflow" 63 | "-Dflowstorm.jarEditorCommand=emacsclient -n +<>:0 <>/<>" 64 | "-Dflowstorm.fileEditorCommand=emacsclient -n +<>:0 <>" 65 | 66 | "-Dvisualvm.display.name=VoceFnExample" 67 | "-Djdk.attach.allowAttachSelf" "-XX:+UnlockDiagnosticVMOptions" "-XX:+DebugNonSafepoints"]} 68 | :deploy {:extra-deps {slipset/deps-deploy {:mvn/version "0.2.2"}} 69 | :exec-fn deps-deploy.deps-deploy/deploy 70 | :exec-args {:installer :remote 71 | :sign-releases? false 72 | :artifact "target/com.shipclojure/simulflow-v0.1.8-alpha.jar"}}}} 73 | -------------------------------------------------------------------------------- /src/simulflow/processors/google.clj: -------------------------------------------------------------------------------- 1 | (ns simulflow.processors.google 2 | (:require 3 | [clojure.core.async.flow :as flow] 4 | [clojure.string :as str] 5 | [hato.client :as http] 6 | [simulflow.schema :as schema] 7 | [simulflow.secrets :refer [secret]] 8 | [simulflow.utils.core :as u] 9 | [simulflow.utils.openai :as uai])) 10 | 11 | (def google-generative-api-url "https://generativelanguage.googleapis.com/v1beta/openai") 12 | (def google-completions-url (str google-generative-api-url "/chat/completions")) 13 | 14 | (comment 15 | ;; Get list of valid models 16 | (->> (http/request {:url (str google-generative-api-url "/models") 17 | :method :get 18 | :headers {"Authorization" (str "Bearer " (secret [:google :api-key]))}}) 19 | :body 20 | u/parse-if-json 21 | :data 22 | (map :id) 23 | (map #(last (str/split % #"/"))) 24 | set)) 25 | 26 | (def model-schema 27 | (schema/flex-enum 28 | {:description "Google llm model identifier" 29 | :error/message "Must be a valid Google LLM model. Try gemini-2.0-flash"} 30 | [:gemini-embedding-exp :gemini-2.0-flash-thinking-exp :learnlm-1.5-pro-experimental :gemini-1.5-pro :gemini-2.0-flash-lite-001 :gemini-1.5-flash 31 | :gemini-2.0-flash-lite-preview :chat-bison-001 :gemini-exp-1206 :gemini-1.5-pro-002 :text-bison-001 :gemini-2.0-flash-lite-preview-02-05 32 | :gemini-1.5-flash-8b-latest :embedding-001 :gemini-2.0-pro-exp :gemini-1.5-flash-8b-001 :gemini-1.0-pro-vision-latest :gemini-2.5-pro-preview-03-25 33 | :learnlm-2.0-flash-experimental :gemini-1.5-flash-002 :gemma-3-27b-it :text-embedding-004 :gemini-1.5-flash-001-tuning :gemini-2.0-flash-lite 34 | :gemini-2.5-flash-preview-04-17-thinking :gemini-1.5-pro-001 :gemini-pro-vision :gemini-1.5-flash-8b-exp-0924 :gemini-2.0-flash-live-001 35 | :gemini-1.5-flash-8b :gemini-2.0-flash-thinking-exp-1219 :gemini-1.5-flash-latest :gemini-1.5-pro-latest :gemma-3-4b-it :embedding-gecko-001 36 | :gemini-2.0-flash-thinking-exp-01-21 :gemini-2.0-pro-exp-02-05 :veo-2.0-generate-001 :gemini-embedding-exp-03-07 :gemini-2.5-pro-exp-03-25 :gemini-1.5-flash-8b-exp-0827 37 | :gemma-3-12b-it :gemini-2.0-flash :gemini-2.5-flash-preview-04-17 :gemini-2.0-flash-001 :gemini-1.5-flash-001 :gemma-3-1b-it :gemini-2.5-pro-preview-05-06 38 | :imagen-3.0-generate-002 :gemini-2.0-flash-exp])) 39 | 40 | (def GoogleLLMConfigSchema 41 | [:map 42 | {:description "Google LLM configuration"} 43 | [:llm/model {:default :gemini-2.0-flash 44 | :description "Google model used for llm inference"} model-schema] 45 | [:google/api-key {:description "Google API key" 46 | :error/message "Invalid Google Api Key provided"} :string] 47 | [:api/completions-url {:default google-completions-url 48 | :optional true 49 | :description "Different completions url for gemini api"} :string]]) 50 | 51 | (comment 52 | 53 | (:body (uai/normal-chat-completion {:api-key (secret [:google :api-key]) 54 | :model :gemini-2.0-flash 55 | :messages [{:role "system" 56 | :content "You are a voice agent operating via phone. Be 57 | concise in your answers. The input you receive comes from a 58 | speech-to-text (transcription) system that isn't always 59 | efficient and may send unclear text. Ask for 60 | clarification when you're unsure what the person said."} 61 | {:role "user" :content "Do you hear me?"}] 62 | :completions-url google-completions-url}))) 63 | 64 | (def describe 65 | {:ins {:in "Channel for incoming context aggregations"} 66 | :outs {:out "Channel where streaming responses will go"} 67 | :params (schema/->describe-parameters GoogleLLMConfigSchema) 68 | :workload :io}) 69 | 70 | (def init! (partial uai/init-llm-processor! GoogleLLMConfigSchema :gemini)) 71 | 72 | (defn transition 73 | [state transition] 74 | (uai/transition-llm-processor state transition)) 75 | 76 | (defn transform 77 | [state in msg] 78 | (uai/transform-llm-processor state in msg :google/api-key :api/completions-url)) 79 | 80 | (defn google-llm-process-fn 81 | ([] describe) 82 | ([params] (init! params)) 83 | ([state trs] 84 | (transition state trs)) 85 | ([state in msg] (transform state in msg))) 86 | 87 | (def google-llm-process (flow/process google-llm-process-fn)) 88 | -------------------------------------------------------------------------------- /src/simulflow/processors/openai.clj: -------------------------------------------------------------------------------- 1 | (ns simulflow.processors.openai 2 | (:require 3 | [clojure.core.async.flow :as flow] 4 | [malli.core :as m] 5 | [simulflow.schema :as schema] 6 | [simulflow.utils.openai :as uai])) 7 | 8 | (def OpenAILLMConfigSchema 9 | [:map 10 | {:description "OpenAI LLM configuration"} 11 | 12 | [:llm/model {:default :gpt-4o-mini} 13 | (schema/flex-enum 14 | {:description "OpenAI model identifier" 15 | :error/message "Must be a valid OpenAI model" 16 | :default "gpt-4o-mini"} 17 | [;; GPT-4o Models (2024-2025) 18 | :gpt-4o 19 | :chatgpt-4o-latest 20 | :gpt-4o-mini 21 | :gpt-4o-audio-preview 22 | :gpt-4o-audio-preview-2024-12-17 23 | :gpt-4o-audio-preview-2024-10-01 24 | :gpt-4o-mini-audio-preview 25 | :gpt-4o-mini-audio-preview-2024-12-17 26 | 27 | ;; GPT-4.1 Models (2025) 28 | :gpt-4.1 29 | :gpt-4.1-mini 30 | :gpt-4.1-nano 31 | :gpt-4.1-2025-04-14 32 | :gpt-4.1-mini-2025-04-14 33 | :gpt-4.1-nano-2025-04-14 34 | 35 | ;; GPT-5 Models (2025) 36 | :gpt-5 37 | :gpt-5-mini 38 | :gpt-5-nano 39 | :gpt-5-chat-latest 40 | :gpt-5-2025-08-07 41 | :gpt-5-mini-2025-08-07 42 | :gpt-5-nano-2025-08-07 43 | 44 | ;; GPT-4 Turbo Models 45 | :gpt-4-turbo 46 | :gpt-4-turbo-2024-04-09 47 | :gpt-4-1106-preview 48 | :gpt-4-0125-preview 49 | :gpt-4-turbo-preview 50 | 51 | ;; GPT-4 Models 52 | :gpt-4 53 | :gpt-4-32k 54 | :gpt-4-vision-preview 55 | 56 | ;; GPT-3.5 Models 57 | :gpt-3.5-turbo 58 | :gpt-3.5-turbo-16k 59 | :gpt-3.5-turbo-1106 60 | :gpt-3.5-turbo-instruct 61 | 62 | ;; O-series Models (2025) 63 | :o3-2025-04-16 64 | :o3-pro 65 | :o4-mini-2025-04-16 66 | ;; Base Models 67 | :babbage-002 68 | :davinci-002])] 69 | [:llm/temperature {:optional true} 70 | [:float 71 | {:description "Sampling temperature (0-2)" 72 | :default 0.7 73 | :min 0.0 74 | :max 2.0}]] 75 | 76 | [:llm/max-tokens {:optional true} 77 | [:int 78 | {:description "Maximum number of tokens to generate" 79 | :min 1}]] 80 | 81 | [:llm/frequency-penalty {:optional true} 82 | [:float 83 | {:description "Frequency penalty (-2.0 to 2.0)" 84 | :min -2.0 85 | :max 2.0}]] 86 | 87 | [:llm/presence-penalty {:optional true} 88 | [:float 89 | {:description "Presence penalty (-2.0 to 2.0)" 90 | :min -2.0 91 | :max 2.0}]] 92 | 93 | [:llm/top-p {:optional true} 94 | [:float 95 | {:description "Nucleus sampling threshold" 96 | :min 0.0 97 | :max 1.0}]] 98 | 99 | [:llm/seed {:optional true} 100 | [:int 101 | {:description "Random seed for deterministic sampling"}]] 102 | 103 | [:llm/max-completion-tokens {:optional true} 104 | [:int 105 | {:description "Maximum tokens in completion" 106 | :min 1}]] 107 | 108 | [:llm/extra {:optional true} 109 | [:map 110 | {:description "Additional model parameters"}]] 111 | 112 | [:openai/api-key 113 | [:string 114 | {:description "OpenAI API key" 115 | :secret true ;; Marks this as sensitive data 116 | :min 40 ;; OpenAI API keys are typically longer 117 | :error/message "Invalid OpenAI API key format"}]] 118 | 119 | [:api/completions-url {:optional true 120 | :default uai/openai-completions-url} :string]]) 121 | 122 | ;; Example validation: 123 | (comment 124 | (require 125 | '[malli.error :as me]) 126 | ;; Valid config 127 | (m/validate OpenAILLMConfigSchema 128 | {:openai/api-key "sk-12312312312312312312312312312312312312312313..."}) 129 | ;; => true 130 | 131 | ;; Invalid model 132 | (-> OpenAILLMConfigSchema 133 | (m/explain {:llm/model "invalid-model" 134 | :openai/api-key "sk-..."}) 135 | me/humanize)) 136 | 137 | (def describe 138 | {:ins {:in "Channel for incoming context aggregations" 139 | :sys-in "Channel for incoming system messages"} 140 | :outs {:out "Channel where streaming responses will go"} 141 | :params (schema/->describe-parameters OpenAILLMConfigSchema) 142 | :workload :io}) 143 | 144 | (def init! (partial uai/init-llm-processor! OpenAILLMConfigSchema :openai)) 145 | 146 | (defn transition 147 | [state transition] 148 | (uai/transition-llm-processor state transition)) 149 | 150 | (defn transform 151 | [state in msg] 152 | (uai/transform-llm-processor state in msg :openai/api-key :api/completions-url)) 153 | 154 | (defn openai-llm-fn 155 | "Multi-arity processor function for OpenAI LLM" 156 | ([] describe) 157 | ([params] (init! params)) 158 | ([state trs] (transition state trs)) 159 | ([state input-port frame] (transform state input-port frame))) 160 | 161 | (def openai-llm-process 162 | "OpenAI LLM processor using separate function approach" 163 | (flow/process openai-llm-fn)) 164 | -------------------------------------------------------------------------------- /src/simulflow/processors/activity_monitor.clj: -------------------------------------------------------------------------------- 1 | (ns simulflow.processors.activity-monitor 2 | "Process to monitor activity on the call. Used to ping user when no activity is detected." 3 | (:require 4 | [clojure.core.async :as a] 5 | [clojure.core.async.flow :as flow] 6 | [simulflow.async :refer [vthread-loop]] 7 | [simulflow.frame :as frame] 8 | [simulflow.schema :as schema] 9 | [taoensso.telemere :as t])) 10 | 11 | (def ActivityMonitorConfigSchema 12 | [:map 13 | [::timeout-ms 14 | {:default 5000 15 | :optional true 16 | :description "Timeout in ms before sending inactivity message. Default 5000ms"} 17 | :int] 18 | [::end-phrase 19 | {:default "Goodbye!" 20 | :optional true 21 | :description "Message for bot to say in order to end the conversation"} 22 | :string] 23 | [::max-pings 24 | {:default 3 25 | :optional true 26 | :description "Maximum number of inactivity pings before ending the conversation"} 27 | :int] 28 | [::ping-phrases 29 | {:default #{"Are you still there?"} 30 | :optional true 31 | :description "Collection (set or vector) with messages to send when inactivity is detected."} 32 | [:or 33 | [:set :string] 34 | [:vector :string]]]]) 35 | 36 | ;; When a frame is received 37 | ;; 1. If that frame is VAD for either bot or user 38 | ;; 1.1 If user speaking frame, set user speaking true and send frame to timer process to reset activity timer 39 | ;; 1.2 If user stopped speaking, set user speaking false and send frame to timer process to reset activity timer 40 | ;; 2.1 If bot speaking frame, set bot speaking true and send frame to timer process to reset activity timer 41 | ;; 2.2 If bot stopped speaking, set bot speaking false and send frame to timer process to reset activity timer 42 | ;; 3. If the timeout for activity has passed and no VAD frame came AND nobody is speaking 43 | ;; 3.1 Increment the activity ping message count 44 | ;; 3.2 If the activity ping message count is bigger or equal to max pings send end conversation message 45 | ;; 4.1 If the timeout for activity has passed and no VAD frame came, but the user is speaking, don't send inactivity ping 46 | ;; 4.2 If the timeout for activity has passed and no VAD frame came, but the bot is speaking, don't send inactivity ping 47 | 48 | (defn speaking? 49 | [state] 50 | (or (::bot-speaking? state) 51 | (::user-speaking? state))) 52 | 53 | (defn transform 54 | [state in msg] 55 | (if (= :sys-in in) 56 | (cond 57 | (frame/user-speech-start? msg) 58 | [(assoc state 59 | ::user-speaking? true 60 | ::ping-count 0) {:timer-process-in [msg]}] 61 | 62 | (frame/user-speech-stop? msg) 63 | [(assoc state ::user-speaking? false) {:timer-process-in [msg]}] 64 | 65 | (frame/bot-speech-start? msg) 66 | [(assoc state ::bot-speaking? true) {:timer-process-in [msg]}] 67 | 68 | (frame/bot-speech-stop? msg) 69 | [(assoc state ::bot-speaking? false) {:timer-process-in [msg]}] 70 | 71 | :else [state]) 72 | (let [ping-count (::ping-count state 0) 73 | max-pings (::max-pings state 0) 74 | ping-phrases-raw (::ping-phrases state #{"Are you still there?"}) 75 | ping-phrases (if (seq ping-phrases-raw) ping-phrases-raw #{"Are you still there?"}) 76 | end-phrase (::end-phrase state "Goodbye!") 77 | now (:now state)] 78 | (cond 79 | (and (= in :timer-process-out) 80 | (::timeout? msg) 81 | (< (inc ping-count) max-pings) 82 | (not (speaking? state))) 83 | [(update-in state [::ping-count] (fnil inc 0)) {:out [(frame/speak-frame (rand-nth (vec ping-phrases)) {:timestamp now})]}] 84 | 85 | (and (= in :timer-process-out) 86 | (::timeout? msg) 87 | (= (inc ping-count) max-pings) 88 | (not (speaking? state))) 89 | [(assoc state ::ping-count 0) {:out [(frame/speak-frame end-phrase {:timestamp now})]}] 90 | 91 | :else [state])))) 92 | 93 | (def describe 94 | {:ins {:sys-in "Channel for system messages"} 95 | :outs {:out "Channel for inactivity prompts"} 96 | :params (schema/->describe-parameters ActivityMonitorConfigSchema)}) 97 | 98 | (defn init! [params] 99 | (let [{::keys [timeout-ms] :as parsed} (schema/parse-with-defaults ActivityMonitorConfigSchema params) 100 | timer-in-ch (a/chan 1024) 101 | timer-out-ch (a/chan 1024)] 102 | (vthread-loop [] 103 | (let [timeout-ch (a/timeout timeout-ms) 104 | [_v c] (a/alts!! [timer-in-ch timeout-ch])] 105 | (when (= c timeout-ch) 106 | (t/log! {:msg "Activity timeout activated!" 107 | :id :activity-monitor 108 | :level :debug}) 109 | (a/>!! timer-out-ch {::timeout? true})) 110 | (recur))) 111 | 112 | (merge {::flow/out-ports {:timer-process-in timer-in-ch} 113 | ::flow/in-ports {:timer-process-out timer-out-ch}} 114 | parsed))) 115 | 116 | (defn transition [{::flow/keys [out-ports in-ports] :as state} transition] 117 | (when (= transition ::flow/stop) 118 | (doseq [port (concat (vals out-ports) (vals in-ports))] 119 | (a/close! port))) 120 | state) 121 | 122 | (defn processor-fn 123 | ([] describe) 124 | ([params] (init! params)) 125 | ([state trs] (transition state trs)) 126 | ([state in msg] (transform state in msg))) 127 | 128 | (def process (flow/process processor-fn)) 129 | -------------------------------------------------------------------------------- /doc/implementation/drafts.org: -------------------------------------------------------------------------------- 1 | #+title: Drafts 2 | #+description: The place where ideas battle it out to become decisions 3 | #+startup: indent 4 | 5 | * Audio out transport 6 | 7 | There are two current modes of audio out transport: 8 | - =speakers= - just write the audio to a audio source line 9 | 10 | - =core.async channel= - put audio frames on a channel to be handled by the 11 | outside world. Even =speakers= could be handled this way, we ship it as a 12 | commodity for people. 13 | 14 | 15 | ** Responsabilities of the audio out 16 | 17 | 18 | *** 1. Buffer audio frames to maintain realtime sending 19 | 20 | This is important because we support bot interruption, therefore when the user 21 | speaks, we need to be able to stop the flow of frames. We cannot rely on the 22 | output "device" to handle that when we send an interrupt signal 23 | 24 | *** 2. Send bot speech events 25 | 26 | The audio out sends bot speech events when the bot started/stopped speaking. 27 | These are important for judging if the bot should be interrupted at certain 28 | times or to understand if there is activity on the call - used for [[file:~/workspace/simulflow/src/simulflow/processors/activity_monitor.clj::(ns simulflow.processors.activity-monitor][activity 29 | monitoring]] among others 30 | 31 | *** 3. Accept system frames that contain serializer and serialize output frames when a serializer exists 32 | This is something that most of the time can be handled on the other side of the 33 | core async channel to limit the responsability of the simulflow core but there 34 | might be cases where this is required here. 35 | 36 | *** 4. Handle interuptions 37 | Discard all new audio frames if it is in an interrupted state and clear out the 38 | audio playback queue if it receives an interrupt-start frame 39 | 40 | ** Current problems 41 | 42 | 1. The logic was created more for =speakers-out= but it is better positioned for 43 | normal =realtime async out=. At the core of it, =speakers-out= is just a 44 | different =init!= that starts a audio source line. The code should be 45 | refactored. 46 | 47 | * Audio in transport 48 | 49 | ** DONE Provide a transport in processor that just takes a in channel and receives in frames on it (might be there already) 50 | CLOSED: [2025-09-03 Wed 09:46] 51 | 52 | * Interruptions - Make the pipeline interruptible either through VAD or Smart Turn detecton 53 | 54 | ** How pipecat handles interruptions 55 | 56 | In pipecat, transport contains both [[file:~/workspace/pipecat/src/pipecat/transports/base_input.py::class BaseInputTransport(FrameProcessor):][in]] & out. When audio comes in, each chunk is 57 | sent to a VAD analizer like Silero and optionally a Smart turn detection model. 58 | Based on these analyzers the BaseTransportInput keeps a =speaking= flag. If 59 | there is a smart turn detector, it's logic for wether the user is 60 | speaking/stopped speaking will take precedence, otherwise the VAD will be the 61 | source of truth. 62 | 63 | When the VAD state changes, (ex: speaking -> not speaking), the pipeline emits 64 | =VADUserStartedSpeakingFrame=/=VADUserStoppedSpeakingFrame=. This happens 65 | regardless in order to have good observation on the pipeline. 66 | 67 | The transport input has a role into starting interruptions *if* there isn't a 68 | interruption strategy created. If one or more *interruption* strategies exists, 69 | the triggering of the interruption logic is defered to use UserContextAggregator 70 | that has access to the current transcriptions coming in. 71 | 72 | *** Interruption flow 73 | 74 | 1. When either the BaseTransportInput or the UserContextAggregator deems an 75 | interruption should start, they emit a frame to do so. BaseTransportInput emits 76 | a =StartInterruptionFrame= and UserContextAggregator emits a 77 | =BotInterruptionFrame= which is sent to the BaseTransportInput who upon 78 | receiving this frame, emits a =StartInterruptionFrame=. This frame does the 79 | following: 80 | - Tells the LLM to cancel in-flight requests and current streaming tokens 81 | - Tells TTS processors to cancel in-flight reqeusts and clear their accumulators 82 | - Tells TransportOut to stop sending AudioOut frames and clear the current 83 | playback queue 84 | - (Possibly) tells BotContextAggregator to drop current sentence assembled or 85 | just cut it short as the user only heard a part of it. 86 | 87 | 2. The pipeline is now in a interrupted state (relevant for TransportOut because 88 | it drops any new AudioOut frames until it receives a =StopInterruptionFrame=) 89 | 90 | 3. When user is deemed to have stopped speaking (by either VAD or Turn taking 91 | model) a =UserStoppedSpeaking= frame is sent. Which will trigger a 92 | =StopInterruptionFrame= if the pipeline supports interuption. This frame will 93 | either be sent by the TransportBaseInput or the UserContextAggregator, based 94 | on the existence of Interruption strategies. 95 | 96 | Important mention here: The LLM, TTS don't keep a =speaking= flag in this 97 | period as they don't care about this state. They only drop their current 98 | activity when a =StartInterruptionFrame= is received but don't handle a 99 | =StopInterruptionFrame= at all. 100 | 101 | ** Interruption flow responsabilities 102 | 103 | 1. VAD on each new chunk 104 | 2. (optional) Turn Detect on each new chunk 105 | 3. Start interruption 106 | 4. Stop interruption 107 | 5. Interruption based on strategies 108 | 109 | Basically the VAD and turn detections can be protocols not processors 110 | -------------------------------------------------------------------------------- /src/simulflow/processors/audio_resampler.clj: -------------------------------------------------------------------------------- 1 | (ns simulflow.processors.audio-resampler 2 | "Audio resampling processor for converting between different sample rates and encodings. 3 | 4 | Transforms audio-input-raw frames from one format to another (e.g., 8kHz μ-law to 16kHz PCM). 5 | Useful for adapting audio from transport layers to transcription requirements." 6 | (:require 7 | [clojure.core.async.flow :as flow] 8 | [simulflow.frame :as frame] 9 | [simulflow.schema :as schema] 10 | [simulflow.utils.audio :as audio] 11 | [taoensso.telemere :as t])) 12 | 13 | ;; Configuration Schema 14 | (def AudioResamplerConfigSchema 15 | [:map 16 | [:audio-resample/source-sample-rate 17 | {:default 8000 18 | :optional true 19 | :description "Input sample rate in Hz"} 20 | schema/SampleRate] 21 | [:audio-resample/target-sample-rate 22 | {:default 16000 23 | :optional true 24 | :description "Output sample rate in Hz"} 25 | schema/SampleRate] 26 | [:audio-resample/source-encoding 27 | {:default :ulaw 28 | :optional true 29 | :description "Input encoding format"} 30 | schema/AudioEncoding] 31 | [:audio-resample/target-encoding 32 | {:default :pcm-signed 33 | :optional true 34 | :description "Output encoding format"} 35 | schema/AudioEncoding] 36 | [:audio-resample/channels 37 | {:default 1 38 | :optional true 39 | :description "Number of audio channels"} 40 | schema/AudioChannels] 41 | [:audio-resample/sample-size-bits 42 | {:default 16 43 | :optional true 44 | :description "Bit depth of samples"} 45 | schema/SampleSizeBits] 46 | [:audio-resample/buffer-size 47 | {:default 1024 48 | :optional true 49 | :description "Audio buffer size in bytes"} 50 | schema/BufferSize] 51 | [:audio-resample/endian 52 | {:default :little-endian 53 | :optional true 54 | :description "Audio byte order (endianness)"} 55 | schema/AudioEndian]]) 56 | 57 | (defn audio-frame? 58 | [frame] 59 | (or 60 | (frame/audio-input-raw? frame) 61 | (frame/audio-output-raw? frame))) 62 | 63 | (defn transform 64 | "Transform function that resamples audio-input-raw frames" 65 | [state input-port frame] 66 | (if (and (= input-port :in) 67 | (audio-frame? frame)) 68 | (let [{:audio-resample/keys [source-sample-rate target-sample-rate 69 | source-encoding target-encoding 70 | channels sample-size-bits buffer-size endian]} state 71 | 72 | audio-data ^bytes (:frame/data frame) 73 | 74 | source-config {:sample-rate source-sample-rate 75 | :encoding source-encoding 76 | :channels channels 77 | :sample-size-bits (if (= source-encoding :ulaw) 8 sample-size-bits) 78 | :buffer-size buffer-size 79 | :endian endian} 80 | 81 | target-config {:sample-rate target-sample-rate 82 | :encoding target-encoding 83 | :channels channels 84 | :sample-size-bits sample-size-bits 85 | :buffer-size buffer-size 86 | :endian endian} 87 | 88 | resampled-data (audio/resample-audio-data audio-data source-config target-config) 89 | 90 | output-frame (assoc frame :frame/data resampled-data)] 91 | 92 | (t/log! {:level :debug 93 | :id :audio-resampler 94 | :msg (format "Resampled audio: %d bytes -> %d bytes (%dkHz %s -> %dkHz %s)" 95 | (alength audio-data) (alength resampled-data) 96 | (/ source-sample-rate 1000) source-encoding 97 | (/ target-sample-rate 1000) target-encoding) 98 | :sample 0.05}) 99 | 100 | [state {:out [output-frame]}]) 101 | 102 | ;; Pass through non-audio frames unchanged 103 | [state {:out [frame]}])) 104 | 105 | (def describe 106 | {:ins {:in "Channel for audio-input-raw frames to be resampled"} 107 | :outs {:out "Channel for resampled audio-input-raw frames"} 108 | :params (schema/->describe-parameters AudioResamplerConfigSchema)}) 109 | 110 | (defn init! 111 | "Initialize the audio resampler with configuration" 112 | [params] 113 | (let [config (schema/parse-with-defaults AudioResamplerConfigSchema params)] 114 | (t/log! {:level :info :id :audio-resampler :data config} 115 | (format "Initialized audio resampler: %dkHz %s -> %dkHz %s" 116 | (/ (:audio-resample/source-sample-rate config) 1000) 117 | (:audio-resample/source-encoding config) 118 | (/ (:audio-resample/target-sample-rate config) 1000) 119 | (:audio-resample/target-encoding config))) 120 | config)) 121 | 122 | (defn transition 123 | "Handle processor lifecycle transitions" 124 | [state trs] 125 | (when (= trs ::flow/stop) 126 | (t/log! {:level :info :id :audio-resampler} "Audio resampler stopped")) 127 | state) 128 | 129 | ;; Multi-arity Processor Function 130 | 131 | (defn audio-resampler-fn 132 | "Multi-arity function following simulflow processor pattern" 133 | ([] describe) 134 | ([params] (init! params)) 135 | ([state trs] (transition state trs)) 136 | ([state input-port frame] (transform state input-port frame))) 137 | 138 | ;; Flow Process 139 | 140 | (def process 141 | "Audio resampling processor for flow integration" 142 | (flow/process audio-resampler-fn)) 143 | -------------------------------------------------------------------------------- /src/simulflow/transport/text_out.clj: -------------------------------------------------------------------------------- 1 | (ns simulflow.transport.text-out 2 | (:refer-clojure :exclude [send]) 3 | (:require 4 | [clojure.core.async :as a] 5 | [clojure.core.async.flow :as flow] 6 | [simulflow.async :refer [vthread-loop]] 7 | [simulflow.frame :as frame] 8 | [taoensso.telemere :as t])) 9 | 10 | (def ^:private default-config 11 | {:text-out/response-prefix "Assistant: " 12 | :text-out/response-suffix "" 13 | :text-out/show-thinking false 14 | :text-out/user-prompt "You: " 15 | :text-out/manage-prompts true}) 16 | 17 | (defn text-output-transform 18 | "Pure transform function - handles text output logic and emits commands" 19 | [state _input-port frame] 20 | (let [{:text-out/keys [response-prefix response-suffix show-thinking 21 | user-prompt]} state] 22 | (cond 23 | ;; LLM response start - show prefix, clean formatting 24 | (frame/llm-full-response-start? frame) 25 | [(assoc state :response-active? true) 26 | {::command [{:command/kind :command/print 27 | :command/data response-prefix}]}] 28 | ;; LLM response end - show suffix and prepare for next input 29 | (frame/llm-full-response-end? frame) 30 | [(assoc state :response-active? false) 31 | (cond-> {::command [{:command/kind :command/print 32 | :command/data response-suffix} 33 | {:command/kind :command/println 34 | :command/data ""} ; newline 35 | {:command/kind :command/print 36 | :command/data user-prompt}]})] 37 | ;; Stream LLM text chunks 38 | (frame/llm-text-chunk? frame) 39 | [state {::command [{:command/kind :command/print 40 | :command/data (:frame/data frame)}]}] 41 | 42 | (frame/speak-frame? frame) 43 | [state {::command [{:command/kind :command/println 44 | :command/data ""} 45 | {:command/kind :command/print 46 | :command/data response-prefix} 47 | {:command/kind :command/println 48 | :command/data (:frame/data frame)} 49 | {:command/kind :command/print 50 | :command/data user-prompt}]}] 51 | ;; Handle tool calls quietly unless in debug mode 52 | (frame/llm-tool-call-chunk? frame) 53 | [state (if show-thinking 54 | {::command [{:command/kind :command/debug 55 | :command/data (str "Tool call: " (:frame/data frame))}]} 56 | {})] 57 | :else [state {}]))) 58 | 59 | (defn text-output-init! 60 | "Initialize text output processor with command handling loop" 61 | [params] 62 | (let [config (merge default-config params) 63 | {:text-out/keys [user-prompt manage-prompts]} config 64 | command-ch (a/chan 32) 65 | running? (atom true) 66 | ;; Close function to be called on stop 67 | close-fn (fn [] 68 | (reset! running? false) 69 | (a/close! command-ch))] 70 | ;; Start command handling loop in virtual thread 71 | (vthread-loop [] 72 | (when @running? 73 | (try 74 | (when-let [command (a/ audio-size chunk-actual-size) 42 | (let [new-audio-size (- audio-size chunk-actual-size) 43 | remaining-audio (byte-array new-audio-size)] 44 | (System/arraycopy audio chunk-actual-size remaining-audio 0 new-audio-size) 45 | (recur remaining-audio (conj chunks chunk))) 46 | ;; No more chunks to process, return final result 47 | (conj chunks chunk)))))) 48 | 49 | (defn audio-splitter-config 50 | "Validate and apply defaults to audio splitter configuration." 51 | [config] 52 | (schema/parse-with-defaults AudioSplitterConfig config)) 53 | 54 | (defn audio-splitter-fn 55 | "Audio splitter processor function with multi-arity support." 56 | ([] {:ins {:in "Channel for raw audio frames"} 57 | :outs {:out "Channel for audio frames split by chunk size"} 58 | :params (schema/->describe-parameters AudioSplitterConfig)}) 59 | ([config] 60 | (audio-splitter-config config)) 61 | ([state _] 62 | ;; Transition function must return the state 63 | state) 64 | ([state _ frame] 65 | (cond 66 | (frame/audio-output-raw? frame) 67 | (let [{:audio.out/keys [duration-ms sample-size-bits]} state 68 | {:keys [audio sample-rate]} (:frame/data frame) 69 | ;; Calculate chunk size based on frame's sample rate and configured duration 70 | chunk-size (audio/audio-chunk-size {:sample-rate sample-rate 71 | :channels 1 ;; All generated audio (AI speech) is mono 72 | :sample-size-bits sample-size-bits 73 | :duration-ms duration-ms})] 74 | (t/log! {:id :audio-splitter 75 | :msg "Received audio output. Splitting into chunks" 76 | :level :debug 77 | :data {:sample-rate sample-rate 78 | :chunk-size chunk-size}}) 79 | (if-let [chunks (split-audio-into-chunks audio chunk-size)] 80 | ;; Create new frames preserving the sample rate from original frame 81 | [state {:out (mapv (fn [chunk-audio] 82 | (frame/audio-output-raw {:audio chunk-audio 83 | :sample-rate sample-rate})) 84 | chunks)}] 85 | [state])) 86 | :else [state]))) 87 | 88 | (def audio-splitter 89 | "Takes in audio-output-raw frames and splits them up into :audio.out/duration-ms 90 | chunks. Chunks are split to achieve realtime streaming." 91 | (flow/process audio-splitter-fn)) 92 | 93 | ;; Backward compatibility 94 | (defaliases 95 | {:src out/realtime-out-describe} 96 | {:src out/realtime-out-init!} 97 | {:src out/realtime-out-transition} 98 | {:alias realtime-out-transform :src out/base-realtime-out-transform} 99 | {:src out/realtime-out-fn} 100 | {:src out/realtime-out-processor} 101 | {:src out/realtime-speakers-out-describe} 102 | {:src out/realtime-speakers-out-init!} 103 | {:alias realtime-speakers-out-transition :src out/realtime-out-transition} 104 | {:alias realtime-speakers-out-transform :src out/base-realtime-out-transform} 105 | {:src out/realtime-speakers-out-fn} 106 | {:src out/realtime-speakers-out-processor} 107 | 108 | ;; twilio in 109 | {:src in/twilio-transport-in} 110 | {:src in/twilio-transport-in-describe} 111 | {:src in/twilio-transport-in-init!} 112 | {:src in/twilio-transport-in-transform} 113 | {:src in/twilio-transport-in-fn} 114 | 115 | ;; mic transport in 116 | {:src in/microphone-transport-in} 117 | {:src in/mic-transport-in-describe} 118 | {:src in/mic-transport-in-init!} 119 | {:src in/mic-transport-in-transform} 120 | {:src in/mic-transport-in-fn} 121 | 122 | ;; async-transform-in 123 | {:src in/async-transport-in-describe} 124 | {:src in/async-transport-in-init!} 125 | {:src in/async-transport-in-fn}) 126 | -------------------------------------------------------------------------------- /src/simulflow/processors/system_frame_router.clj: -------------------------------------------------------------------------------- 1 | (ns simulflow.processors.system-frame-router 2 | "Simple system frame router that collects frames on :sys-in and broadcasts on :sys-out. 3 | 4 | This processor acts as a fan-out hub for system frames within a single flow, 5 | eliminating the need for N×(N-1) connections between system frame producers and consumers." 6 | (:require 7 | [clojure.core.async.flow :as flow] 8 | [clojure.datafy :as datafy] 9 | [simulflow.frame :as frame] 10 | [taoensso.telemere :as t])) 11 | 12 | (defn system-frame-router-fn 13 | "Creates a system frame router processor. 14 | 15 | This processor: 16 | 1. Receives all frames on its :sys-in port 17 | 2. Forwards system frames to its :sys-out port 18 | 3. Drops non-system frames with optional logging 19 | 20 | Usage in flow: 21 | :system-router {:proc system-frame-router} 22 | 23 | Connections: 24 | [[:producer :sys-out] [:system-router :sys-in]] 25 | [[:system-router :sys-out] [:consumer1 :sys-in]] 26 | [[:system-router :sys-out] [:consumer2 :sys-in]] 27 | [[:system-router :sys-out] [:consumer3 :sys-in]]" 28 | 29 | ([] 30 | ;; 0-arity: describe the processor 31 | {:ins {:sys-in "System frames from producers"} 32 | :outs {:sys-out "System frames broadcasted to consumers"} 33 | :params {:log-dropped? "Log when non-system frames are dropped"}}) 34 | 35 | ([{:keys [log-dropped?] :or {log-dropped? false}}] 36 | ;; 1-arity: init processor state 37 | {:log-dropped? log-dropped?}) 38 | 39 | ([state _transition] 40 | ;; 2-arity: handle state transitions (not used) 41 | state) 42 | 43 | ([state input-port frame] 44 | ;; 3-arity: transform function - main routing logic 45 | (cond 46 | (and (= input-port :sys-in) (some? frame)) 47 | (if (frame/system-frame? frame) 48 | ;; Forward system 49 | [state {:sys-out [frame]}] 50 | ;; Drop non-system frames 51 | (do 52 | (when (:log-dropped? state) 53 | (t/log! {:level :debug :id :system-frame-router :frame-type (:frame/type frame)} 54 | "Dropped non-system frame")) 55 | [state])) 56 | 57 | :else 58 | ;; Unknown input port or nil frame - no output 59 | [state {}]))) 60 | 61 | (def system-frame-router 62 | "Convenience function that wraps the system-frame-router in a flow/process. 63 | 64 | This is the standard way to create the processor for use in flows." 65 | (flow/process system-frame-router-fn)) 66 | 67 | (defn contains-system-router? 68 | "Returns true if the flow-config contains the system-frame-router-process" 69 | [flow-config] 70 | (reduce-kv (fn [_ _ v] 71 | (if (= (:proc v) system-frame-router) 72 | (reduced true) 73 | false)) 74 | false 75 | flow-config)) 76 | 77 | (defn generate-system-router-connections 78 | "Given a flow config's processor map, analyzes each processor's :proc with clojure.datafy/datafy 79 | to check if it has :sys-in and :sys-out channels in its :desc. Returns a vector 80 | of connection pairs that should be added to connect processors to the system-router. 81 | 82 | Assumes the system-router processor is named :system-router in the flow config. 83 | 84 | Returns connections in the format: 85 | [;; System frame producers -> router 86 | [[:producer-with-sys-out :sys-out] [:system-router :sys-in]] 87 | ;; Router -> system frame consumers 88 | [[:system-router :sys-out] [:consumer-with-sys-in :sys-in]]]" 89 | [flow-config-processors] 90 | (let [processor-analyses (for [[proc-name {:keys [proc]}] flow-config-processors 91 | :when (and proc (not= proc-name :system-router))] 92 | (try 93 | (let [desc (-> proc datafy/datafy :desc)] 94 | {:name proc-name 95 | :has-sys-in? (contains? (:ins desc) :sys-in) 96 | :has-sys-out? (contains? (:outs desc) :sys-out)}) 97 | (catch Exception e 98 | {:name proc-name 99 | :has-sys-in? false 100 | :has-sys-out? false 101 | :error (.getMessage e)}))) 102 | 103 | sys-out-producers (filter :has-sys-out? processor-analyses) 104 | sys-in-consumers (filter :has-sys-in? processor-analyses) 105 | 106 | producer-connections (map (fn [{:keys [name]}] 107 | [[name :sys-out] [:system-router :sys-in]]) 108 | sys-out-producers) 109 | 110 | consumer-connections (map (fn [{:keys [name]}] 111 | [[:system-router :sys-out] [name :sys-in]]) 112 | sys-in-consumers)] 113 | 114 | (vec (concat producer-connections consumer-connections)))) 115 | 116 | (comment) 117 | ;; Usage in a flow - eliminates complex system frame connections 118 | 119 | ;; Before (complex N×N connections): 120 | ;; [[:activity-monitor :out] [:context-aggregator :in]] 121 | ;; [[:activity-monitor :out] [:tts :in]] 122 | ;; [[:transport-out :out] [:activity-monitor :in]] 123 | ;; [[:transport-out :out] [:context-aggregator :in]] 124 | 125 | ;; After (simple fan-out via system router): 126 | ;; {:procs {:system-router {:proc system-frame-router-process} 127 | ;; :activity-monitor {:proc activity-monitor-process} 128 | ;; :context-aggregator {:proc context-aggregator-process} 129 | ;; :transport-out {:proc transport-out-process} 130 | ;; :tts {:proc tts-process}} 131 | ;; 132 | ;; :conns [;; System frame producers -> router 133 | ;; [[:activity-monitor :out] [:system-router :sys-in]] 134 | ;; [[:transport-out :out] [:system-router :sys-in]] 135 | ;; 136 | ;; ;; Router -> system frame consumers 137 | ;; [[:system-router :sys-out] [:activity-monitor :sys-in]] 138 | ;; [[:system-router :sys-out] [:context-aggregator :sys-in]] 139 | ;; [[:system-router :sys-out] [:tts :sys-in]]]} 140 | -------------------------------------------------------------------------------- /examples/src/simulflow_examples/scenario_example.clj: -------------------------------------------------------------------------------- 1 | (ns simulflow-examples.scenario-example 2 | (:require 3 | [clojure.core.async :as a] 4 | [clojure.core.async.flow :as flow] 5 | [simulflow-examples.local :as local] 6 | [simulflow.scenario-manager :as sm] 7 | [taoensso.telemere :as t])) 8 | 9 | (def config 10 | {:initial-node :start 11 | :nodes 12 | {:start 13 | ;; Role messages dictate how the AI should behave. Ideally :role-messages 14 | ;; should be present on the :initial-node as they persist for the rest of the conversation 15 | {:run-llm? true ;; Run the AI when this node is set. We don't run inference as we will run pre-defined speech 16 | :role-messages [{:role :system 17 | :content "You are a restaurant reservation assistant for La Maison, an upscale French restaurant. You must ALWAYS use one of the available functions to progress the conversation. This is a phone conversations and your responses will be converted to audio. Avoid outputting special characters and emojis. Be casual and friendly."}] 18 | ;; task-messages signify the current task the AI has. Each node requires a 19 | ;; task 20 | :task-messages [{:role :system 21 | :content "Warmly greet the customer and ask how many people are in their party."}] 22 | :pre-actions [{:type :tts-say 23 | :text "Hello! Welcome to La Maison! How many people are in your party?"}] 24 | :functions [{:type :function 25 | :function 26 | {:name "record_party_size" 27 | :handler (fn [{:keys [size]}] size) 28 | :description "Record the number of people in the party" 29 | :parameters 30 | {:type :object 31 | :properties 32 | {:size {:type :integer 33 | :description "The number of people that will dine." 34 | :minimum 1 35 | :maximum 12}} 36 | :required [:size]} 37 | ;; transition-to dictates the next node the AI will go to 38 | :transition-to :get-time}}]} 39 | :get-time 40 | {:task-messages [{:role :system 41 | :content "Ask what time they'd like to dine. Restaurant is open 5 PM to 10 PM. After they provide a time, confirm it's within operating hours before recording. Use 24-hour format for internal recording (e.g., 17:00 for 5 PM)."}] 42 | :functions [{:type :function 43 | :function {:name "record_time" 44 | :handler (fn [{:keys [time]}] time) 45 | :description "Record the requested time" 46 | :parameters {:type :object 47 | :properties {:time {:type :string 48 | :pattern "^(17|18|19|20|21|22):([0-5][0-9])$" 49 | :description "Reservation time in 24-hour format (17:00-22:00)"}} 50 | :required [:time]} 51 | :transition-to :confirm}}]} 52 | 53 | :confirm 54 | {:task-messages [{:role :system 55 | :content "Confirm the reservation details and end the conversation. Say back to the client the details of the reservation: \"Ok! So a reservation for X people at Y PM. Is this corret?\" "}] 56 | :functions [{:type :function 57 | :function {:name "end" 58 | :description "End the conversation" 59 | :parameters {:type :object, :properties {}} 60 | :transition-to :end}}]} 61 | :end {:task-messages [{:role :system, :content "Thank them and end the conversation."}] 62 | :functions [] 63 | :pre-actions [{:type :end-conversation}]}}}) 64 | 65 | (defn scenario-example 66 | "A scenario is a predefined, highly structured conversation. LLM performance 67 | degrades when it has a big complex prompt to enact, so to ensure a consistent 68 | output use scenarios that transition the LLM into a new scenario node with a clear 69 | instruction for the current node." 70 | ([] 71 | (scenario-example {:initial-node :start})) 72 | ([{:keys [initial-node]}] 73 | (let [flow (local/make-local-flow 74 | {;; Don't add any context because the scenario will handle that 75 | :llm/context {:messages [] 76 | :tools []} 77 | :language :en 78 | 79 | ;; add gateway process for scenario to inject frames 80 | :extra-procs {:scenario {:proc (flow/process #'sm/scenario-in-process)}} 81 | 82 | :extra-conns [[[:scenario :sys-out] [:tts :in]] 83 | [[:scenario :sys-out] [:context-aggregator :sys-in]]]}) 84 | 85 | s (sm/scenario-manager {:flow flow 86 | :flow-in-coord [:scenario :scenario-in] ;; scenario-manager will inject frames through this channel 87 | :scenario-config (assoc config :initial-node initial-node)})] 88 | 89 | {:flow flow 90 | :scenario s}))) 91 | 92 | (comment 93 | 94 | (def call-in-progress? (atom false)) 95 | ;; create flow & scenario 96 | (def s (scenario-example {:initial-node :start})) 97 | 98 | ;; Start local ai flow - starts paused 99 | (let [{:keys [report-chan error-chan]} (flow/start (:flow s))] 100 | ;; start the scenario by setting initial node 101 | (sm/start (:scenario s)) 102 | ;; Resume local ai -> you can now speak with the AI 103 | (flow/resume (:flow s)) 104 | (reset! call-in-progress? true) 105 | ;; Monitor report & error channels 106 | (a/thread 107 | (loop [] 108 | (when @call-in-progress? 109 | (when-let [[msg c] (a/alts!! [report-chan error-chan])] 110 | (when (map? msg) 111 | (t/log! {:level :debug :id (if (= c error-chan) :error :report)} msg)) 112 | (recur)))))) 113 | 114 | ;; Stop the conversation 115 | (do 116 | (flow/stop (:flow s)) 117 | (reset! call-in-progress? false)) 118 | 119 | ,) 120 | -------------------------------------------------------------------------------- /examples/src/simulflow_examples/local_with_mute_filter.clj: -------------------------------------------------------------------------------- 1 | (ns simulflow-examples.local-with-mute-filter 2 | "This example demonstrates muting user input handling​ When the LLM is responding to 3 | the user, if the user says something while the bot is speaking it will not be processed." 4 | (:require 5 | [clojure.core.async :as a] 6 | [clojure.core.async.flow :as flow] 7 | [simulflow.async :refer [vthread-loop]] 8 | [simulflow.filters.mute :as mute-filter] 9 | [simulflow.processors.deepgram :as deepgram] 10 | [simulflow.processors.elevenlabs :as xi] 11 | [simulflow.processors.llm-context-aggregator :as context] 12 | [simulflow.processors.openai :as openai] 13 | [simulflow.processors.system-frame-router :as system-router] 14 | [simulflow.secrets :refer [secret]] 15 | [simulflow.transport :as transport] 16 | [simulflow.transport.in :as transport-in] 17 | [simulflow.transport.out :as transport-out] 18 | [taoensso.telemere :as t])) 19 | 20 | (def llm-context 21 | {:messages 22 | [{:role "system" 23 | :content "You are a voice agent operating via phone. Be 24 | concise in your answers. The input you receive comes from a 25 | speech-to-text (transcription) system that isn't always 26 | efficient and may send unclear text. Ask for 27 | clarification when you're unsure what the person said."}] 28 | :tools 29 | [{:type :function 30 | :function 31 | {:name "get_weather" 32 | :handler (fn [{:keys [town]}] (str "The weather in " town " is 17 degrees celsius")) 33 | :description "Get the current weather of a location" 34 | :parameters {:type :object 35 | :required [:town] 36 | :properties {:town {:type :string 37 | :description "Town for which to retrieve the current weather"}} 38 | :additionalProperties false} 39 | :strict true}}]}) 40 | 41 | (def chunk-duration 20) 42 | 43 | (def flow-processors 44 | {;; Capture audio from microphone and send raw-audio-input frames further in the pipeline 45 | :transport-in {:proc transport-in/microphone-transport-in 46 | :args {:vad/analyser :vad.analyser/silero 47 | :pipeline/supports-interrupt? false}} 48 | ;; raw-audio-input -> transcription frames 49 | :transcriptor {:proc deepgram/deepgram-processor 50 | :args {:transcription/api-key (secret [:deepgram :api-key]) 51 | :transcription/interim-results? true 52 | :transcription/punctuate? false 53 | ;; We use silero for computing VAD 54 | :transcription/vad-events? false 55 | :transcription/smart-format? true 56 | :transcription/model :nova-2 57 | :transcription/utterance-end-ms 1000 58 | :transcription/language :en}} 59 | 60 | ;; user transcription & llm message frames -> llm-context frames 61 | ;; responsible for keeping the full conversation history 62 | :context-aggregator {:proc context/context-aggregator 63 | :args {:llm/context llm-context}} 64 | 65 | ;; Takes llm-context frames and produces new llm-text-chunk & llm-tool-call-chunk frames 66 | :llm {:proc openai/openai-llm-process 67 | :args {:openai/api-key (secret [:openai :new-api-sk]) 68 | :llm/model :gpt-4.1-mini}} 69 | 70 | ;; llm-text-chunk & llm-tool-call-chunk -> llm-context-messages-append frames 71 | :assistant-context-assembler {:proc context/assistant-context-assembler 72 | :args {}} 73 | 74 | ;; llm-text-chunk -> sentence speak frames (faster for text to speech) 75 | :llm-sentence-assembler {:proc context/llm-sentence-assembler} 76 | 77 | ;; speak-frames -> audio-output-raw frames 78 | :tts {:proc xi/elevenlabs-tts-process 79 | :args {:elevenlabs/api-key (secret [:elevenlabs :api-key]) 80 | :elevenlabs/model-id "eleven_flash_v2_5" 81 | :elevenlabs/voice-id (secret [:elevenlabs :voice-id]) 82 | :voice/stability 0.5 83 | :voice/similarity-boost 0.8 84 | :voice/use-speaker-boost? true 85 | :pipeline/language :en}} 86 | 87 | ;; audio-output-raw -> smaller audio-output-raw frames (used for sending audio in realtime) 88 | :audio-splitter {:proc transport/audio-splitter 89 | :args {:audio.out/duration-ms chunk-duration}} 90 | 91 | ;; speakers out 92 | :transport-out {:proc transport-out/realtime-speakers-out-processor 93 | :args {:audio.out/sending-interval chunk-duration 94 | :audio.out/duration-ms chunk-duration}} 95 | 96 | :mute-filter {:proc mute-filter/process 97 | :args {:mute/strategies #{:mute.strategy/bot-speech}}} 98 | ;; Fan out of system frames to processors that need it 99 | :system-router {:proc system-router/system-frame-router}}) 100 | 101 | (def flow-conns (into (system-router/generate-system-router-connections flow-processors) 102 | [[[:transport-in :out] [:transcriptor :in]] 103 | 104 | [[:transcriptor :out] [:context-aggregator :in]] 105 | [[:context-aggregator :out] [:llm :in]] 106 | 107 | ;; Aggregate full context 108 | [[:llm :out] [:assistant-context-assembler :in]] 109 | [[:assistant-context-assembler :out] [:context-aggregator :in]] 110 | 111 | ;; Assemble sentence by sentence for fast speech 112 | [[:llm :out] [:llm-sentence-assembler :in]] 113 | 114 | [[:tts :out] [:audio-splitter :in]] 115 | [[:audio-splitter :out] [:transport-out :in]]])) 116 | 117 | (def flow-config 118 | {:procs flow-processors 119 | :conns flow-conns}) 120 | 121 | (def g (flow/create-flow flow-config)) 122 | 123 | (defonce flow-started? (atom false)) 124 | 125 | (comment 126 | 127 | ;; Start local ai flow - starts paused 128 | (let [{:keys [report-chan error-chan]} (flow/start g)] 129 | (reset! flow-started? true) 130 | ;; Resume local ai -> you can now speak with the AI 131 | (flow/resume g) 132 | (vthread-loop [] 133 | (when @flow-started? 134 | (when-let [[msg c] (a/alts!! [report-chan error-chan])] 135 | (when (map? msg) 136 | (t/log! (cond-> {:level :debug :id (if (= c error-chan) :error :report)} 137 | (= c error-chan) (assoc :error msg)) msg)) 138 | (recur))))) 139 | 140 | ;; Stop the conversation 141 | (do 142 | (flow/stop g) 143 | (reset! flow-started? false)) 144 | 145 | ,) 146 | -------------------------------------------------------------------------------- /examples/src/simulflow_examples/scenario_example_system_router_mute_filter.clj: -------------------------------------------------------------------------------- 1 | (ns simulflow-examples.scenario-example-system-router-mute-filter 2 | (:require 3 | [clojure.core.async :as a] 4 | [clojure.core.async.flow :as flow] 5 | [simulflow-examples.scenario-example :refer [config]] 6 | [simulflow.async :refer [vthread-loop]] 7 | [simulflow.filters.mute :as mute-filter] 8 | [simulflow.processors.deepgram :as deepgram] 9 | [simulflow.processors.elevenlabs :as xi] 10 | [simulflow.processors.llm-context-aggregator :as context] 11 | [simulflow.processors.openai :as openai] 12 | [simulflow.processors.system-frame-router :as system-router] 13 | [simulflow.scenario-manager :as sm] 14 | [simulflow.secrets :refer [secret]] 15 | [simulflow.transport :as transport] 16 | [simulflow.transport.in :as transport-in] 17 | [simulflow.transport.out :as transport-out] 18 | [taoensso.telemere :as t])) 19 | 20 | (def llm-context 21 | {:messages 22 | [{:role "system" 23 | :content "You are a voice agent operating via phone. Be 24 | concise in your answers. The input you receive comes from a 25 | speech-to-text (transcription) system that isn't always 26 | efficient and may send unclear text. Ask for 27 | clarification when you're unsure what the person said."}] 28 | :tools 29 | [{:type :function 30 | :function 31 | {:name "get_weather" 32 | :handler (fn [{:keys [town]}] (str "The weather in " town " is 17 degrees celsius")) 33 | :description "Get the current weather of a location" 34 | :parameters {:type :object 35 | :required [:town] 36 | :properties {:town {:type :string 37 | :description "Town for which to retrieve the current weather"}} 38 | :additionalProperties false} 39 | :strict true}}]}) 40 | 41 | (def chunk-duration 20) 42 | 43 | (def flow-processors 44 | {;; Capture audio from microphone and send raw-audio-input frames further in the pipeline 45 | :transport-in {:proc transport-in/microphone-transport-in 46 | :args {:vad/analyser :vad.analyser/silero 47 | :pipeline/supports-interrupt? false}} 48 | ;; raw-audio-input -> transcription frames 49 | :transcriptor {:proc deepgram/deepgram-processor 50 | :args {:transcription/api-key (secret [:deepgram :api-key]) 51 | :transcription/interim-results? true 52 | :transcription/punctuate? false 53 | ;; We use silero for computing VAD 54 | :transcription/vad-events? false 55 | :transcription/smart-format? true 56 | :transcription/model :nova-2 57 | :transcription/utterance-end-ms 1000 58 | :transcription/language :en}} 59 | 60 | ;; user transcription & llm message frames -> llm-context frames 61 | ;; responsible for keeping the full conversation history 62 | :context-aggregator {:proc context/context-aggregator 63 | :args {:llm/context llm-context}} 64 | 65 | ;; Takes llm-context frames and produces new llm-text-chunk & llm-tool-call-chunk frames 66 | :llm {:proc openai/openai-llm-process 67 | :args {:openai/api-key (secret [:openai :new-api-sk]) 68 | :llm/model :gpt-4.1-mini}} 69 | 70 | ;; llm-text-chunk & llm-tool-call-chunk -> llm-context-messages-append frames 71 | :assistant-context-assembler {:proc context/assistant-context-assembler 72 | :args {}} 73 | 74 | ;; llm-text-chunk -> sentence speak frames (faster for text to speech) 75 | :llm-sentence-assembler {:proc context/llm-sentence-assembler} 76 | 77 | ;; speak-frames -> audio-output-raw frames 78 | :tts {:proc xi/elevenlabs-tts-process 79 | :args {:elevenlabs/api-key (secret [:elevenlabs :api-key]) 80 | :elevenlabs/model-id "eleven_flash_v2_5" 81 | :elevenlabs/voice-id (secret [:elevenlabs :voice-id]) 82 | :voice/stability 0.5 83 | :voice/similarity-boost 0.8 84 | :voice/use-speaker-boost? true 85 | :pipeline/language :en}} 86 | 87 | ;; audio-output-raw -> smaller audio-output-raw frames (used for sending audio in realtime) 88 | :audio-splitter {:proc transport/audio-splitter 89 | :args {:audio.out/duration-ms chunk-duration}} 90 | 91 | ;; speakers out 92 | :transport-out {:proc transport-out/realtime-speakers-out-processor 93 | :args {:audio.out/sending-interval chunk-duration 94 | :audio.out/duration-ms chunk-duration}} 95 | 96 | :mute-filter {:proc mute-filter/process 97 | :args {:mute/strategies #{:mute.strategy/bot-speech}}} 98 | 99 | :scenario-in {:proc sm/scenario-in-process} 100 | ;; Fan out of system frames to processors that need it 101 | :system-router {:proc system-router/system-frame-router}}) 102 | 103 | (def flow-conns (into (system-router/generate-system-router-connections flow-processors) 104 | [[[:transport-in :out] [:transcriptor :in]] 105 | 106 | [[:transcriptor :out] [:context-aggregator :in]] 107 | [[:context-aggregator :out] [:llm :in]] 108 | 109 | ;; Aggregate full context 110 | [[:llm :out] [:assistant-context-assembler :in]] 111 | [[:assistant-context-assembler :out] [:context-aggregator :in]] 112 | 113 | ;; Assemble sentence by sentence for fast speech 114 | [[:llm :out] [:llm-sentence-assembler :in]] 115 | 116 | [[:tts :out] [:audio-splitter :in]] 117 | [[:audio-splitter :out] [:transport-out :in]]])) 118 | 119 | (def flow-config 120 | {:procs flow-processors 121 | :conns flow-conns}) 122 | 123 | (def g (flow/create-flow flow-config)) 124 | 125 | (defonce flow-started? (atom false)) 126 | 127 | (def s (sm/scenario-manager {:flow g 128 | :scenario-config config 129 | :flow-in-coord [:scenario-in :scenario-in]})) 130 | 131 | (comment 132 | 133 | ;; Start local ai flow - starts paused 134 | (let [{:keys [report-chan error-chan]} (flow/start g)] 135 | (reset! flow-started? true) 136 | (sm/start s) 137 | ;; Resume local ai -> you can now speak with the AI 138 | (flow/resume g) 139 | (vthread-loop [] 140 | (when @flow-started? 141 | (when-let [[msg c] (a/alts!! [report-chan error-chan])] 142 | (when (map? msg) 143 | (t/log! (cond-> {:level :debug :id (if (= c error-chan) :error :report)} 144 | (= c error-chan) (assoc :error msg)) msg)) 145 | (recur))))) 146 | 147 | ;; Stop the conversation 148 | (do 149 | (flow/stop g) 150 | (reset! flow-started? false)) 151 | 152 | ,) 153 | -------------------------------------------------------------------------------- /examples/src/simulflow_examples/local_w_interruption_support.clj: -------------------------------------------------------------------------------- 1 | (ns simulflow-examples.local-w-interruption-support 2 | "This example demonstrates interruption handling​ When the LLM is responding to 3 | the user, if the user says something, the current playback is interrupted and 4 | processors that have interruption support drop their activity until the user stops speaking." 5 | (:require 6 | [clojure.core.async :as a] 7 | [clojure.core.async.flow :as flow] 8 | [simulflow.async :refer [vthread-loop]] 9 | [simulflow.processors.activity-monitor :as activity-monitor] 10 | [simulflow.processors.deepgram :as deepgram] 11 | [simulflow.processors.elevenlabs :as xi] 12 | [simulflow.processors.llm-context-aggregator :as context] 13 | [simulflow.processors.openai :as openai] 14 | [simulflow.processors.system-frame-router :as system-router] 15 | [simulflow.secrets :refer [secret]] 16 | [simulflow.transport :as transport] 17 | [simulflow.transport.in :as transport-in] 18 | [simulflow.transport.out :as transport-out] 19 | [taoensso.telemere :as t])) 20 | 21 | (def llm-context 22 | {:messages 23 | [{:role "system" 24 | :content "You are a voice agent operating via phone. Be 25 | concise in your answers. The input you receive comes from a 26 | speech-to-text (transcription) system that isn't always 27 | efficient and may send unclear text. Ask for 28 | clarification when you're unsure what the person said."}] 29 | :tools 30 | [{:type :function 31 | :function 32 | {:name "get_weather" 33 | :handler (fn [{:keys [town]}] (str "The weather in " town " is 17 degrees celsius")) 34 | :description "Get the current weather of a location" 35 | :parameters {:type :object 36 | :required [:town] 37 | :properties {:town {:type :string 38 | :description "Town for which to retrieve the current weather"}} 39 | :additionalProperties false} 40 | :strict true}}]}) 41 | 42 | (def chunk-duration 20) 43 | 44 | (def flow-processors 45 | {;; Capture audio from microphone and send raw-audio-input frames further in the pipeline 46 | :transport-in {:proc transport-in/microphone-transport-in 47 | :args {:vad/analyser :vad.analyser/silero 48 | :pipeline/supports-interrupt? true}} 49 | ;; raw-audio-input -> transcription frames 50 | :transcriptor {:proc deepgram/deepgram-processor 51 | :args {:transcription/api-key (secret [:deepgram :api-key]) 52 | :transcription/interim-results? true 53 | :transcription/punctuate? false 54 | ;; We use silero for computing VAD 55 | :transcription/vad-events? false 56 | :transcription/smart-format? true 57 | :transcription/model :nova-2 58 | :transcription/utterance-end-ms 1000 59 | :transcription/language :en}} 60 | 61 | ;; user transcription & llm message frames -> llm-context frames 62 | ;; responsible for keeping the full conversation history 63 | :context-aggregator {:proc context/context-aggregator 64 | :args {:llm/context llm-context}} 65 | 66 | ;; Takes llm-context frames and produces new llm-text-chunk & llm-tool-call-chunk frames 67 | :llm {:proc openai/openai-llm-process 68 | :args {:openai/api-key (secret [:openai :new-api-sk]) 69 | :llm/model :gpt-4.1-mini}} 70 | 71 | ;; llm-text-chunk & llm-tool-call-chunk -> llm-context-messages-append frames 72 | :assistant-context-assembler {:proc context/assistant-context-assembler 73 | :args {}} 74 | 75 | ;; llm-text-chunk -> sentence speak frames (faster for text to speech) 76 | :llm-sentence-assembler {:proc context/llm-sentence-assembler} 77 | 78 | ;; speak-frames -> audio-output-raw frames 79 | :tts {:proc xi/elevenlabs-tts-process 80 | :args {:elevenlabs/api-key (secret [:elevenlabs :api-key]) 81 | :elevenlabs/model-id "eleven_flash_v2_5" 82 | :elevenlabs/voice-id (secret [:elevenlabs :voice-id]) 83 | :voice/stability 0.5 84 | :voice/similarity-boost 0.8 85 | :voice/use-speaker-boost? true 86 | :pipeline/language :en}} 87 | 88 | ;; audio-output-raw -> smaller audio-output-raw frames (used for sending audio in realtime) 89 | :audio-splitter {:proc transport/audio-splitter 90 | :args {:audio.out/duration-ms chunk-duration}} 91 | 92 | ;; speakers out 93 | :transport-out {:proc transport-out/realtime-speakers-out-processor 94 | :args {:audio.out/sending-interval chunk-duration 95 | :audio.out/duration-ms chunk-duration}} 96 | 97 | :activity-monitor {:proc activity-monitor/process 98 | :args {::activity-monitor/timeout-ms 5000}} 99 | :system-router {:proc system-router/system-frame-router}}) 100 | 101 | (def flow-conns (into (system-router/generate-system-router-connections flow-processors) 102 | [[[:transport-in :out] [:transcriptor :in]] 103 | 104 | [[:transcriptor :out] [:context-aggregator :in]] 105 | [[:context-aggregator :out] [:llm :in]] 106 | 107 | ;; Aggregate full context 108 | [[:llm :out] [:assistant-context-assembler :in]] 109 | [[:assistant-context-assembler :out] [:context-aggregator :in]] 110 | 111 | ;; Assemble sentence by sentence for fast speech 112 | [[:llm :out] [:llm-sentence-assembler :in]] 113 | 114 | [[:tts :out] [:audio-splitter :in]] 115 | [[:audio-splitter :out] [:transport-out :in]] 116 | 117 | ;; Activity detection 118 | [[:activity-monitor :out] [:context-aggregator :in]] 119 | [[:activity-monitor :out] [:tts :in]]])) 120 | 121 | (def flow-config 122 | {:procs flow-processors 123 | :conns flow-conns}) 124 | 125 | (def g (flow/create-flow flow-config)) 126 | 127 | (defonce flow-started? (atom false)) 128 | 129 | (comment 130 | 131 | ;; Start local ai flow - starts paused 132 | (let [{:keys [report-chan error-chan]} (flow/start g)] 133 | (reset! flow-started? true) 134 | ;; Resume local ai -> you can now speak with the AI 135 | (flow/resume g) 136 | (vthread-loop [] 137 | (when @flow-started? 138 | (when-let [[msg c] (a/alts!! [report-chan error-chan])] 139 | (when (map? msg) 140 | (t/log! (cond-> {:level :debug :id (if (= c error-chan) :error :report)} 141 | (= c error-chan) (assoc :error msg)) msg)) 142 | (recur))))) 143 | 144 | ;; Stop the conversation 145 | (do 146 | (flow/stop g) 147 | (reset! flow-started? false)) 148 | 149 | ,) 150 | -------------------------------------------------------------------------------- /src/simulflow/utils/request.clj: -------------------------------------------------------------------------------- 1 | (ns simulflow.utils.request 2 | "Taken from https://github.com/wkok/openai-clojure/blob/main/src/wkok/openai_clojure/sse.clj." 3 | {:no-doc true} 4 | (:require 5 | [clojure.core.async :as a] 6 | [clojure.string :as string] 7 | [hato.client :as http] 8 | [hato.middleware :as hm] 9 | [simulflow.async :refer [vthread vthread-loop]] 10 | [simulflow.utils.core :as u] 11 | [taoensso.telemere :as t]) 12 | (:import 13 | (java.io InputStream))) 14 | 15 | (def event-mask (re-pattern "(?s).+?\n\n")) 16 | 17 | (defn deliver-events 18 | [events {:keys [on-next]}] 19 | (when on-next 20 | (vthread-loop [] 21 | (when-let [event (a/!! events %)) es)) 89 | (recur (drop (apply + (map #(count (.getBytes ^String %)) es)) 90 | next-byte-coll)) 91 | 92 | ;; Output stream closed, exiting read-loop 93 | (do 94 | (t/log! {:level :trace :msg "SSE output stream closed during event sending"}) 95 | nil))) 96 | 97 | (do 98 | (t/log! {:level :trace :msg "SSE no events found in data chunk"}) 99 | (recur next-byte-coll))))))) 100 | (finally 101 | (when close? 102 | (a/close! events)) 103 | (.close event-stream)))) 104 | 105 | events)) 106 | 107 | (defn sse-request 108 | "Process streamed results. 109 | If on-next callback provided, then read from channel and call the callback. 110 | Returns a response with the core.async channel as the body" 111 | [{:keys [params] :as ctx}] 112 | (let [events (sse-events ctx)] 113 | (deliver-events events params) 114 | {:status 200 115 | :body events})) 116 | 117 | (defn wrap-trace 118 | "Middleware that allows the user to supply a trace function that 119 | will receive the raw request & response as arguments. 120 | See: https://github.com/gnarroway/hato?tab=readme-ov-file#custom-middleware" 121 | [trace] 122 | (fn [client] 123 | (fn 124 | ([req] 125 | (let [resp (client req)] 126 | (trace req resp) 127 | resp)) 128 | ([req respond raise] 129 | (client req 130 | #(respond (do (trace req %) 131 | %)) 132 | raise))))) 133 | 134 | (defn with-trace-middleware 135 | "The default list of middleware hato uses for wrapping requests but 136 | with added wrap-trace in the correct position to allow tracing of error messages." 137 | [trace] 138 | [hm/wrap-request-timing 139 | 140 | hm/wrap-query-params 141 | hm/wrap-basic-auth 142 | hm/wrap-oauth 143 | hm/wrap-user-info 144 | hm/wrap-url 145 | 146 | hm/wrap-decompression 147 | hm/wrap-output-coercion 148 | 149 | (wrap-trace trace) 150 | 151 | hm/wrap-exceptions 152 | hm/wrap-accept 153 | hm/wrap-accept-encoding 154 | hm/wrap-multipart 155 | 156 | hm/wrap-content-type 157 | hm/wrap-form-params 158 | hm/wrap-nested-params 159 | hm/wrap-method]) 160 | 161 | (comment 162 | (require '[simulflow.secrets :refer [secret]]) 163 | 164 | (http/request {:url "https://api.openai.com/v1/chat/completions" 165 | :headers {"Authorization" (str "Bearer " (secret [:openai :new-api-sk])) 166 | "Content-Type" "application/json"} 167 | 168 | :method :post 169 | :body (u/json-str {:messages [{:role "system" :content "You are a helpful assistant"} 170 | {:role "user" :content "What is the capital of France?"}] 171 | :model "gpt-4o-mini"})}) 172 | 173 | (def res (sse-request 174 | {:request {:url "https://api.openai.com/v1/chat/completions" 175 | :headers {"Authorization" (str "Bearer " (secret [:openai :new-api-sk])) 176 | "Content-Type" "application/json"} 177 | 178 | :method :post 179 | :body (u/json-str {:messages [{:role "system" :content "You are a helpful assistant"} 180 | {:role "user" :content "What is the capital of France?"}] 181 | :stream true 182 | :model "gpt-4o-mini"})} 183 | :params {:stream/close? true}})) 184 | 185 | res 186 | 187 | (def ch (:body res)) 188 | 189 | (a/go-loop [] 190 | (println "Taking from result") 191 | (when-let [res (a/ transcription frames 67 | :transcriptor {:proc deepgram/deepgram-processor 68 | :args {:transcription/api-key (secret [:deepgram :api-key]) 69 | :transcription/interim-results? true 70 | :transcription/punctuate? false 71 | :transcription/vad-events? false 72 | :transcription/smart-format? true 73 | :transcription/model :nova-2 74 | :transcription/utterance-end-ms 1000 75 | :transcription/language language}} 76 | 77 | ;; user transcription & llm message frames -> llm-context frames 78 | ;; responsible for keeping the full conversation history 79 | :context-aggregator {:proc context/context-aggregator 80 | :args {:llm/context llm-context 81 | :aggregator/debug? debug?}} 82 | 83 | ;; Takes llm-context frames and produces new llm-text-chunk & llm-tool-call-chunk frames 84 | :llm {:proc openai/openai-llm-process 85 | :args {:openai/api-key (secret [:openai :new-api-sk]) 86 | :llm/model :gpt-4.1-mini}} 87 | 88 | ;; llm-text-chunk & llm-tool-call-chunk -> llm-context-messages-append frames 89 | :assistant-context-assembler {:proc context/assistant-context-assembler 90 | :args {:debug? debug?}} 91 | 92 | ;; llm-text-chunk -> sentence speak frames (faster for text to speech) 93 | :llm-sentence-assembler {:proc context/llm-sentence-assembler} 94 | 95 | ;; speak-frames -> audio-output-raw frames 96 | :tts {:proc xi/elevenlabs-tts-process 97 | :args {:elevenlabs/api-key (secret [:elevenlabs :api-key]) 98 | :elevenlabs/model-id "eleven_flash_v2_5" 99 | :elevenlabs/voice-id (secret [:elevenlabs :voice-id]) 100 | :voice/stability 0.5 101 | :voice/similarity-boost 0.8 102 | :voice/use-speaker-boost? true 103 | :pipeline/language language}} 104 | 105 | ;; audio-output-raw -> smaller audio-output-raw frames (used for sending audio in realtime) 106 | :audio-splitter {:proc transport/audio-splitter 107 | :args {:audio.out/duration-ms chunk-duration-ms}} 108 | 109 | ;; speakers out 110 | :transport-out {:proc transport-out/realtime-speakers-out-processor 111 | :args {:audio.out/sending-interval chunk-duration-ms 112 | :audio.out/duration-ms chunk-duration-ms}} 113 | 114 | :activity-monitor {:proc activity-monitor/process 115 | :args {::activity-monitor/timeout-ms 5000}}} 116 | extra-procs) 117 | :conns (concat 118 | [[[:transport-in :out] [:transcriptor :in]] 119 | 120 | [[:transcriptor :out] [:context-aggregator :in]] 121 | [[:transport-in :sys-out] [:context-aggregator :sys-in]] 122 | [[:context-aggregator :out] [:llm :in]] 123 | 124 | ;; Aggregate full context 125 | [[:llm :out] [:assistant-context-assembler :in]] 126 | [[:assistant-context-assembler :out] [:context-aggregator :in]] 127 | 128 | ;; Assemble sentence by sentence for fast speech 129 | [[:llm :out] [:llm-sentence-assembler :in]] 130 | [[:llm-sentence-assembler :sys-out] [:tts :sys-in]] 131 | 132 | [[:tts :out] [:audio-splitter :in]] 133 | [[:audio-splitter :out] [:transport-out :in]] 134 | 135 | ;; Activity detection 136 | [[:transport-out :sys-out] [:activity-monitor :sys-in]] 137 | [[:transport-in :sys-out] [:activity-monitor :sys-in]] 138 | [[:transcriptor :sys-out] [:activity-monitor :sys-in]] 139 | [[:activity-monitor :out] [:context-aggregator :in]] 140 | [[:activity-monitor :out] [:tts :in]]] 141 | extra-conns)}))) 142 | 143 | (comment 144 | 145 | (def local-ai (make-local-flow)) 146 | 147 | (defonce flow-started? (atom false)) 148 | 149 | ;; Start local ai flow - starts paused 150 | (let [{:keys [report-chan error-chan]} (flow/start local-ai)] 151 | (reset! flow-started? true) 152 | ;; Resume local ai -> you can now speak with the AI 153 | (flow/resume local-ai) 154 | (vthread-loop [] 155 | (when @flow-started? 156 | (when-let [[msg c] (a/alts!! [report-chan error-chan])] 157 | (when (map? msg) 158 | (t/log! (cond-> {:level :debug :id (if (= c error-chan) :error :report)} 159 | (= c error-chan) (assoc :error msg)) msg)) 160 | (recur))))) 161 | 162 | ;; Stop the conversation 163 | (do 164 | (flow/stop local-ai) 165 | (reset! flow-started? false)) 166 | 167 | ,) 168 | -------------------------------------------------------------------------------- /src/simulflow/processors/groq.clj: -------------------------------------------------------------------------------- 1 | (ns simulflow.processors.groq 2 | (:require 3 | [clojure.core.async :as a] 4 | [clojure.core.async.flow :as flow] 5 | [hato.client :as http] 6 | [malli.core :as m] 7 | [malli.transform :as mt] 8 | [simulflow.frame :as frame] 9 | [simulflow.schema :as schema] 10 | [simulflow.utils.core :as u] 11 | [simulflow.utils.request :as request] 12 | [taoensso.telemere :as t])) 13 | 14 | (def groq-api-url "https://api.groq.com/openai/v1") 15 | (def groq-completions-url (str groq-api-url "/chat/completions")) 16 | 17 | (defn stream-groq-chat-completion 18 | [{:keys [api-key messages tools model]}] 19 | (:body (request/sse-request {:request {:url groq-completions-url 20 | :headers {"Authorization" (str "Bearer " api-key) 21 | "Content-Type" "application/json"} 22 | 23 | :method :post 24 | :body (u/json-str (cond-> {:messages messages 25 | :stream true 26 | :model model} 27 | (pos? (count tools)) (assoc :tools tools)))} 28 | :params {:stream/close? true}}))) 29 | 30 | (defn normal-chat-completion 31 | [{:keys [api-key messages tools model]}] 32 | (http/request {:url groq-completions-url 33 | :headers {"Authorization" (str "Bearer " api-key) 34 | "Content-Type" "application/json"} 35 | 36 | :throw-on-error? false 37 | :method :post 38 | :body (u/json-str (cond-> {:messages messages 39 | :stream true 40 | :model model} 41 | (pos? (count tools)) (assoc :tools tools)))})) 42 | 43 | (comment 44 | 45 | (map u/token-content (a/> (http/get (str groq-api-url "/models") 54 | {:headers {"Authorization" (str "Bearer " (secret [:groq :api-key])) 55 | "Content-Type" "application/json"}}) 56 | :body 57 | u/parse-if-json 58 | :data 59 | (map :id))) 60 | 61 | (def delta (comp :delta first :choices)) 62 | 63 | (def GroqLLMConfigSchema 64 | [:map 65 | {:closed true 66 | :description "Groq LLM configuration"} 67 | 68 | [:llm/model 69 | (schema/flex-enum 70 | {:description "Groq model identifier" 71 | :error/message "Must be a valid Groq model" 72 | :default "llama-3.3-70b-versatile"} 73 | ["llama-3.2-3b-preview" 74 | "llama-3.1-8b-instant" 75 | "llama-3.3-70b-versatile" 76 | "llama-3.2-11b-vision-preview" 77 | "whisper-large-v3-turbo" 78 | "llama-3.1-70b-versatile" 79 | "llama3-8b-8192" 80 | "llama3-70b-8192" 81 | "llama-guard-3-8b" 82 | "whisper-large-v3" 83 | "llama-3.2-1b-preview" 84 | "mixtral-8x7b-32768" 85 | "gemma2-9b-it" 86 | "llama-3.2-90b-vision-preview" 87 | "llama-3.3-70b-specdec" 88 | "distil-whisper-large-v3-en"])] 89 | 90 | [:groq/api-key 91 | [:string 92 | {:description "Groq API key" 93 | :secret true ;; Marks this as sensitive data 94 | :min 40 ;; Groq API keys are typically longer 95 | :error/message "Invalid Groq API key format"}]]]) 96 | 97 | (defn flow-do-completion! 98 | "Handle completion requests for Groq LLM models" 99 | [state out-c context] 100 | (let [{:llm/keys [model] :groq/keys [api-key]} state] 101 | ;; Start request only when the last message in the context is by the user 102 | 103 | (a/>!! out-c (frame/llm-full-response-start true)) 104 | (let [stream-ch (try (stream-groq-chat-completion (merge {:model model 105 | :api-key api-key 106 | :messages (:messages context) 107 | :tools (mapv u/->tool-fn (:tools context))})) 108 | (catch Exception e 109 | (t/log! {:level :error :id :groq} ["Stream completion error" e])))] 110 | 111 | (a/go-loop [] 112 | (when-let [chunk (a/! out-c (frame/llm-full-response-end true)) 116 | (do 117 | (if-let [tool-call (first (:tool_calls d))] 118 | (do 119 | (t/log! {:level :debug :id :groq} ["Sending tool call" tool-call]) 120 | (a/>! out-c (frame/llm-tool-call-chunk tool-call))) 121 | (when-let [c (:content d)] 122 | (a/>! out-c (frame/llm-text-chunk c)))) 123 | (recur))))))))) 124 | 125 | (defn tool-result-adapter 126 | "Transform tool results to the groq format" 127 | [{:keys [result tool-id fname]}] 128 | {:role :tool 129 | :name fname 130 | :content (if (string? result) result (u/json-str result)) 131 | :tool_call_id tool-id}) 132 | 133 | (def groq-llm-process 134 | (flow/process 135 | (flow/map->step 136 | {:describe (fn [] {:ins {:in "Channel for incoming context aggregations"} 137 | :outs {:out "Channel where streaming responses will go"} 138 | :params {:llm/model "Groq model used" 139 | :groq/api-key "Groq Api key" 140 | :llm/temperature "Optional temperature parameter for the llm inference" 141 | :llm/max-tokens "Optional max tokens to generate" 142 | :llm/presence-penalty "Optional (-2.0 to 2.0)" 143 | :llm/top-p "Optional nucleus sampling threshold" 144 | :llm/seed "Optional seed used for deterministic sampling" 145 | :llm/max-completion-tokens "Optional Max tokens in completion"} 146 | :workload :io}) 147 | 148 | :transition (fn [{::flow/keys [in-ports out-ports] :as state} transition] 149 | (when (= transition ::flow/stop) 150 | (doseq [port (concat (vals in-ports) (vals out-ports))] 151 | (a/close! port))) 152 | state) 153 | :init (fn [params] 154 | (let [state (m/decode GroqLLMConfigSchema params mt/default-value-transformer) 155 | llm-write (a/chan 100) 156 | llm-read (a/chan 1024) 157 | write-to-llm #(loop [] 158 | (if-let [frame (a/ m :channel :alternatives first :transcript)) 43 | 44 | (defn final-transcript? 45 | [m] 46 | (:is_final m)) 47 | 48 | (defn interim-transcript? 49 | [m] 50 | (let [trsc (transcript m)] 51 | (and (not (final-transcript? m)) 52 | (string? trsc) 53 | (not= "" trsc)))) 54 | 55 | (defn speech-started-event? 56 | [m] 57 | (= (:type m) "SpeechStarted")) 58 | 59 | (defn utterance-end-event? 60 | [m] 61 | (= (:type m) "UtteranceEnd")) 62 | 63 | (def close-connection-payload (u/json-str {:type "CloseStream"})) 64 | 65 | (def keep-alive-payload (u/json-str {:type "KeepAlive"})) 66 | 67 | (defn deepgram-event->frames 68 | [event & {:keys [send-interrupt? supports-interrupt?]}] 69 | (let [trsc (transcript event)] 70 | (cond 71 | (speech-started-event? event) 72 | [(frame/user-speech-start true)] 73 | 74 | (utterance-end-event? event) 75 | (if supports-interrupt? 76 | [(frame/user-speech-stop true) (frame/control-interrupt-stop true)] 77 | [(frame/user-speech-stop true)]) 78 | 79 | (final-transcript? event) 80 | [(frame/transcription trsc)] 81 | 82 | (interim-transcript? event) 83 | (if (and supports-interrupt? send-interrupt?) 84 | ;; Deepgram sends a lot of speech start events, many of which are false 85 | ;; positives. They have a disrupting effect on the pipeline, so instead 86 | ;; of sending interrupt-start frames on speech-start events, we send it 87 | ;; on the first interim transcription event AFTER a speech start - this 88 | ;; tends to be a better indicator of speech start 89 | [(frame/transcription-interim trsc) (frame/control-interrupt-start true)] 90 | [(frame/transcription-interim trsc)])))) 91 | 92 | (def DeepgramConfigSchema 93 | [:map 94 | [:transcription/api-key :string] 95 | [:transcription/model {:default :nova-2-general} 96 | (flex-enum (into [:nova-2] (map #(str "nova-2-" %) #{"general" "meeting" "phonecall" "voicemail" "finance" "conversationalai" "video" "medical" "drivethru" "automotive" "atc"})))] 97 | [:transcription/interim-results? {:default false 98 | :optional true} :boolean] 99 | [:transcription/smart-format? {:default true 100 | :optional true} :boolean] 101 | [:transcription/profanity-filter? {:default true 102 | :optional true} :boolean] 103 | [:transcription/supports-interrupt? {:optional true 104 | :default false} :boolean] 105 | [:transcription/vad-events? {:default false 106 | :optional true} :boolean] 107 | [:transcription/utterance-end-ms {:optional true} :int] 108 | [:transcription/language {:default :en} schema/Language] 109 | [:transcription/punctuate? {:default false} :boolean]]) 110 | 111 | (def DeepgramConfig 112 | [:and 113 | DeepgramConfigSchema 114 | ;; if smart-format is true, no need for punctuate 115 | [:fn {:error/message "When :transcription/utterance-end-ms is provided, :transcription/interim-results? must be true. More details here: 116 | https://developers.deepgram.com/docs/understanding-end-of-speech-detection#using-utteranceend"} 117 | (fn [{:transcription/keys [utterance-end-ms interim-results?]}] 118 | (if (some? utterance-end-ms) 119 | interim-results? 120 | true))] 121 | [:fn {:error/message "When :transcription/smart-format? is true, :transcription/punctuate? must be false. More details here: https://developers.deepgram.com/docs/smart-format#enable-feature"} 122 | (fn [{:transcription/keys [smart-format? punctuate?]}] 123 | (not (and smart-format? punctuate?)))]]) 124 | 125 | (def describe 126 | {:ins {:sys-in "Channel for system messages that take priority" 127 | :in "Channel for audio input frames (from transport-in)"} 128 | :outs {:sys-out "Channel for system messages that have priority" 129 | :out "Channel on which transcription frames are put"} 130 | :params (schema/->describe-parameters DeepgramConfigSchema) 131 | :workload :io}) 132 | 133 | (defn init! 134 | [args] 135 | ;; Validate configuration 136 | (let [validated-args (schema/parse-with-defaults DeepgramConfig args) 137 | websocket-url (make-websocket-url validated-args) 138 | ws-read-chan (a/chan 1024) 139 | ws-write-chan (a/chan 1024) 140 | alive? (atom true) 141 | conn-config {:headers {"Authorization" (str "Token " (:transcription/api-key validated-args))} 142 | :on-open (fn [_] 143 | (t/log! {:level :info :id :deepgram} "Websocket connection open")) 144 | :on-message (fn [_ws ^HeapCharBuffer data _last?] 145 | (a/put! ws-read-chan (str data))) 146 | :on-error (fn [_ e] 147 | (t/log! {:level :error :id :deepgram} ["Websocket error" e])) 148 | :on-close (fn [_ws code reason] 149 | (reset! alive? false) 150 | (t/log! {:level :info :id :deepgram} ["Websocket connection closed" "Code:" code "Reason:" reason]))} 151 | 152 | _ (t/log! {:level :info :id :deepgram} ["Connecting to websocket" websocket-url]) 153 | ws-conn @(ws/websocket websocket-url conn-config)] 154 | 155 | ;; Audio message processing loop 156 | (vthread-loop [] 157 | (when @alive? 158 | (when-let [msg (a/frames m)] 196 | [state (apply frame/send frames)]) 197 | (cond 198 | (frame/audio-input-raw? msg) [state {:ws-write [msg]}] 199 | :else [state]))) 200 | 201 | (defn processor-fn 202 | ([] describe) 203 | ([params] (init! params)) 204 | ([state trs] (transition state trs)) 205 | ([state in-name msg] (transform state in-name msg))) 206 | 207 | (def deepgram-processor (flow/process processor-fn)) 208 | -------------------------------------------------------------------------------- /src/simulflow/utils/openai.clj: -------------------------------------------------------------------------------- 1 | (ns simulflow.utils.openai 2 | "Common logic for openai format requests. Many LLM providers use openai format 3 | for their APIs. This NS keeps common logic for those providers." 4 | (:require 5 | [clojure.core.async :as a] 6 | [clojure.core.async.flow :as flow] 7 | [hato.client :as http] 8 | [simulflow.async :refer [vthread-loop]] 9 | [simulflow.command :as command] 10 | [simulflow.frame :as frame] 11 | [simulflow.schema :as schema] 12 | [simulflow.utils.core :as u] 13 | [simulflow.utils.request :as request] 14 | [taoensso.telemere :as t]) 15 | (:import 16 | (clojure.lang ExceptionInfo))) 17 | 18 | (def response-chunk-delta 19 | "Retrieve the delta part of a streaming completion response" 20 | (comp :delta first :choices)) 21 | 22 | (defn handle-completion-request! 23 | "Handle completion requests for OpenAI LLM models" 24 | [in-c out-c] 25 | (vthread-loop [] 26 | (when-let [chunk (a/!! out-c (frame/llm-full-response-end true)) 30 | (do 31 | (if-let [tool-call (first (:tool_calls d))] 32 | (a/>!! out-c (frame/llm-tool-call-chunk tool-call)) 33 | ;; normal text completion 34 | (when-let [c (:content d)] 35 | (a/>!! out-c (frame/llm-text-chunk c)))) 36 | (recur))))))) 37 | 38 | (def openai-completions-url "https://api.openai.com/v1/chat/completions") 39 | 40 | (defn stream-chat-completion 41 | [{:keys [api-key messages tools model response-format completions-url] 42 | :or {model "gpt-4o-mini" 43 | completions-url openai-completions-url}}] 44 | (:body (request/sse-request {:request {:url completions-url 45 | :headers {"Authorization" (str "Bearer " api-key) 46 | "Content-Type" "application/json"} 47 | 48 | :method :post 49 | :body (u/json-str (cond-> {:messages messages 50 | :stream true 51 | :response_format response-format 52 | :model model} 53 | (pos? (count tools)) (assoc :tools tools)))} 54 | :params {:stream/close? true}}))) 55 | 56 | (defn normal-chat-completion 57 | [{:keys [api-key messages tools model response-format stream completions-url] 58 | :or {model "gpt-4o-mini" 59 | completions-url openai-completions-url 60 | stream false}}] 61 | (http/request {:url completions-url 62 | :headers {"Authorization" (str "Bearer " api-key) 63 | "Content-Type" "application/json"} 64 | 65 | :throw-on-error? false 66 | :method :post 67 | :body (u/json-str (cond-> {:messages messages 68 | :stream stream 69 | :response_format response-format 70 | :model model} 71 | (pos? (count tools)) (assoc :tools tools)))})) 72 | 73 | ;; Common processor functions 74 | (defn vthread-pipe-response-with-interrupt 75 | "Common function to handle streaming response from LLM" 76 | [{:keys [in-ch out-ch interrupt-ch on-end] :as data}] 77 | (t/log! {:msg "Piping out result" :data data :level :trace :id :llm-processor}) 78 | (vthread-loop [] 79 | (let [[val port] (a/alts!! [interrupt-ch in-ch] :priority true)] 80 | (t/log! {:level :trace 81 | :msg "Piping current val" 82 | :data {:val val 83 | :chan (if (= port in-ch) :in-ch :interrupt-ch)} 84 | :id :llm-processor}) 85 | 86 | (if (and (= port in-ch) val) 87 | (do 88 | (a/>!! out-ch val) 89 | (recur)) 90 | ;; No more data or interruption, call on-end and exit 91 | (when (fn? on-end) 92 | (on-end)))))) 93 | 94 | (defn init-llm-processor! 95 | "Common initialization function for OpenAI-compatible LLM processors" 96 | [schema log-id params] 97 | (let [parsed-config (schema/parse-with-defaults schema params) 98 | llm-write (a/chan 100) 99 | llm-read (a/chan 1024) 100 | interrupt-ch (a/chan 10) 101 | request-in-progress? (atom false)] 102 | (vthread-loop [] 103 | (when-let [command (a/!! interrupt-ch command)) 123 | 124 | nil) 125 | (catch ExceptionInfo e 126 | (t/log! {:level :error :id log-id :error e} "Error processing command") 127 | (when (= (:command/kind command) :command/sse-request) 128 | (let [data (ex-data e) 129 | body (slurp (:body data))] 130 | (t/log! {:level :error :id log-id :data {:body body}} "Error details"))))) 131 | (recur))) 132 | 133 | (merge parsed-config 134 | {::flow/in-ports {::llm-read llm-read} 135 | ::flow/out-ports {::llm-write llm-write} 136 | ::interrupt-ch interrupt-ch}))) 137 | 138 | (defn transition-llm-processor 139 | "Common transition function for LLM processors" 140 | [{::flow/keys [in-ports out-ports] :as state} transition] 141 | (when (= transition ::flow/stop) 142 | (doseq [port (concat (vals in-ports) (vals out-ports))] 143 | (a/close! port)) 144 | (when-let [c (::interrupt-ch state)] 145 | (a/close! c))) 146 | state) 147 | 148 | (defn transform-handle-llm-response 149 | "Common function to handle the streaming response from the LLM" 150 | [state msg] 151 | (let [d (response-chunk-delta msg) 152 | tool-call (first (:tool_calls d)) 153 | c (:content d)] 154 | (cond 155 | (= msg :done) 156 | [state (frame/send (frame/llm-full-response-end true))] 157 | 158 | tool-call 159 | [state (frame/send (frame/llm-tool-call-chunk tool-call))] 160 | 161 | c 162 | [state (frame/send (frame/llm-text-chunk c))] 163 | 164 | :else [state]))) 165 | 166 | (defn transform-llm-context 167 | "Common function to transform LLM context into SSE request" 168 | [state context-frame api-key-key completions-url-key] 169 | (let [context-data (:frame/data context-frame) 170 | {:llm/keys [model]} state 171 | api-key (get state api-key-key) 172 | completions-url (get state completions-url-key) 173 | tools (mapv u/->tool-fn (:tools context-data)) 174 | request-body (cond-> {:messages (:messages context-data) 175 | :stream true 176 | :model model} 177 | (pos? (count tools)) (assoc :tools tools))] 178 | [state {::llm-write [(command/sse-request-command {:url completions-url 179 | :method :post 180 | :body request-body 181 | :headers {"Authorization" (str "Bearer " api-key) 182 | "Content-Type" "application/json"}})] 183 | :out [(frame/llm-full-response-start true)]}])) 184 | 185 | (defn transform-llm-processor 186 | "Common transform function for LLM processors" 187 | [state in msg api-key-key completions-url-key] 188 | (cond 189 | (= in ::llm-read) 190 | (transform-handle-llm-response state msg) 191 | 192 | (frame/llm-context? msg) 193 | (transform-llm-context state msg api-key-key completions-url-key) 194 | 195 | (frame/control-interrupt-start? msg) 196 | [state {:llm-write [{:command/kind :command/interrupt-request}]}] 197 | 198 | :else 199 | [state {}])) 200 | -------------------------------------------------------------------------------- /examples/src/simulflow_examples/text_chat.clj: -------------------------------------------------------------------------------- 1 | (ns simulflow-examples.text-chat 2 | {:clj-reload/no-unload true} 3 | (:require 4 | [clojure.core.async :as a] 5 | [clojure.core.async.flow :as flow] 6 | [simulflow.processors.llm-context-aggregator :as context] 7 | [simulflow.processors.openai :as openai] 8 | [simulflow.scenario-manager :as sm] 9 | [simulflow.secrets :refer [secret]] 10 | [simulflow.transport.text-in :as text-in] 11 | [simulflow.transport.text-out :as text-out] 12 | [simulflow.utils.core :as u] 13 | [taoensso.telemere :as t])) 14 | 15 | ;; Set log level to reduce noise during chat 16 | (t/set-min-level! :warn) 17 | 18 | (defn text-chat-flow-config 19 | "Text-based chat flow using stdin/stdout instead of voice I/O. 20 | 21 | This example demonstrates how to interact with simulflow through text: 22 | - User types messages instead of speaking 23 | - LLM responses are streamed to console as they're generated 24 | - Input is blocked while LLM is responding 25 | - Clean prompt management handled by text-output processor 26 | - Uses existing frame types: user-speech-start/transcription/user-speech-stop sequence 27 | 28 | For a convenient CLI version, use: bin/chat 29 | " 30 | ([] (text-chat-flow-config {})) 31 | ([{:keys [llm/context debug? extra-procs extra-conns] 32 | :or {context {:messages 33 | [{:role "system" 34 | :content "You are a helpful AI assistant. Be concise and conversational. 35 | You are communicating through text chat."}] 36 | :tools 37 | [{:type :function 38 | :function 39 | {:name "get_weather" 40 | :handler (fn [{:keys [town]}] 41 | (str "The weather in " town " is 17 degrees celsius")) 42 | :description "Get the current weather of a location" 43 | :parameters {:type :object 44 | :required [:town] 45 | :properties {:town {:type :string 46 | :description "Town for which to retrieve the current weather"}} 47 | :additionalProperties false} 48 | :strict true}} 49 | {:type :function 50 | :function 51 | {:name "quit_chat" 52 | :handler (fn [_] 53 | (println "\nGoodbye!") 54 | (System/exit 0)) 55 | :description "Quit the chat session" 56 | :parameters {:type :object 57 | :properties {} 58 | :additionalProperties false} 59 | :strict true}}]} 60 | debug? false}}] 61 | 62 | {:procs 63 | (u/deep-merge 64 | {;; Read from stdin and emit user-speech-start/transcription/user-speech-stop sequence 65 | :text-input {:proc text-in/text-input-process 66 | :args {}} 67 | ;; Handle conversation context and user speech aggregation 68 | :context-aggregator {:proc context/context-aggregator 69 | :args {:llm/context context 70 | :aggregator/debug? debug?}} 71 | ;; Generate LLM responses with streaming 72 | :llm {:proc openai/openai-llm-process 73 | :args {:openai/api-key (secret [:openai :api-key]) 74 | :llm/model :gpt-4.1-mini}} 75 | ;; Handle assistant message assembly for context 76 | :assistant-context-assembler {:proc context/assistant-context-assembler 77 | :args {:debug? debug?}} 78 | ;; Stream LLM output and manage prompts 79 | :text-output {:proc text-out/text-output-process 80 | :args {:text-out/response-prefix "Assistant: " 81 | :text-out/response-suffix "" 82 | :text-out/show-thinking debug? 83 | :text-out/user-prompt "You: " 84 | :text-out/manage-prompts true}}} 85 | extra-procs) 86 | :conns (concat 87 | [;; Main conversation flow 88 | [[:text-input :out] [:context-aggregator :in]] 89 | [[:context-aggregator :out] [:llm :in]] 90 | ;; Stream LLM responses to output for clean formatting 91 | [[:llm :out] [:text-output :in]] 92 | ;; Assemble assistant context for conversation history 93 | [[:llm :out] [:assistant-context-assembler :in]] 94 | [[:assistant-context-assembler :out] [:context-aggregator :in]] 95 | ;; System frame routing for input blocking/unblocking 96 | ;; LLM emits llm-full-response-start/end frames to :out, which are system frames 97 | [[:llm :out] [:text-input :sys-in]] 98 | ;; System frames for lifecycle management 99 | [[:text-input :sys-out] [:context-aggregator :sys-in]]] 100 | extra-conns)})) 101 | 102 | (defn scenario-example 103 | "A scenario is a predefined, highly structured conversation. LLM performance 104 | degrades when it has a big complex prompt to enact, so to ensure a consistent 105 | output use scenarios that transition the LLM into a new scenario node with a clear 106 | instruction for the current node." 107 | [{:keys [initial-node scenario-config]}] 108 | (let [flow (flow/create-flow 109 | (text-chat-flow-config 110 | {;; Don't add any context because the scenario will handle that 111 | :llm/context {:messages [] 112 | :tools []} 113 | :language :en 114 | 115 | ;; add gateway process for scenario to inject frames 116 | :extra-procs {:scenario {:proc (flow/process #'sm/scenario-in-process)}} 117 | 118 | :extra-conns [[[:scenario :sys-out] [:text-output :in]] 119 | [[:scenario :sys-out] [:context-aggregator :sys-in]]]})) 120 | 121 | s (when scenario-config 122 | (sm/scenario-manager 123 | {:flow flow 124 | :flow-in-coord [:scenario :scenario-in] ;; scenario-manager will inject frames through this channel 125 | :scenario-config (cond-> scenario-config 126 | initial-node (assoc :initial-node initial-node))}))] 127 | 128 | {:flow flow 129 | :scenario s})) 130 | 131 | (defn start-text-chat! 132 | "Start a text-based chat session with the simulflow agent. 133 | 134 | Usage: 135 | (start-text-chat!) ; Regular chat 136 | (start-text-chat! {:scenario? true}) ; Scenario example 137 | 138 | Then type messages and press Enter. The assistant will respond with streaming text. 139 | Input is blocked while the assistant is responding. 140 | 141 | Type messages with 'quit' to exit via the quit_chat function. 142 | 143 | CLI Alternative: Use bin/chat for a convenient command-line interface." 144 | ([] (start-text-chat! {})) 145 | ([{:keys [scenario?] :as config}] 146 | (println "🤖 Starting Simulflow Text Chat...") 147 | (println "💡 Type your messages and press Enter") 148 | (println "⏸️ Input is blocked while assistant responds") 149 | (println "🚪 Ask to 'quit chat' to exit") 150 | (println "🔗 CLI version available at: bin/chat") 151 | (println (apply str (repeat 50 "="))) 152 | ;; Create flow based on scenario choice 153 | (let [{:keys [flow scenario]} (if scenario? 154 | (scenario-example {:scenario-config scenario-example/config}) 155 | {:flow (flow/create-flow (text-chat-flow-config config)) :scenario nil}) 156 | {:keys [report-chan error-chan]} (flow/start flow)] 157 | ;; Start error/report monitoring in background 158 | (future 159 | (loop [] 160 | (when-let [[msg c] (a/alts!! [report-chan error-chan] :default nil)] 161 | (when msg 162 | (when (= c error-chan) 163 | (println "\n❌ Error:" msg)) 164 | (recur))))) 165 | ;; Resume the flow to begin processing 166 | (flow/resume flow) 167 | (when scenario (sm/start scenario)) 168 | ;; Return flow for manual control if needed 169 | flow))) 170 | 171 | (defn -main 172 | "Main entry point for text chat demo" 173 | [& args] 174 | (let [debug? (some #(= % "--debug") args) 175 | config (cond-> {} 176 | debug? (assoc :debug? true) 177 | (some #(= % "--scenario") args) (assoc :scenario? true))] 178 | (when debug? 179 | (t/set-min-level! :debug)) 180 | (start-text-chat! config))) 181 | 182 | (comment 183 | ;; Interactive usage examples: 184 | ;; 1. Basic usage 185 | (def chat (start-text-chat!)) 186 | ;; Now type in the console: 187 | ;; You: Hello, how are you? 188 | ;; Assistant: Hello! I'm doing well, thank you for asking... 189 | ;; 2. With debug mode 190 | (def debug-chat (start-text-chat! {:debug? true})) 191 | ;; 3. With custom system message 192 | (def expert-chat (start-text-chat! 193 | {:llm-context 194 | {:messages 195 | [{:role "system" 196 | :content "You are a technical expert in Clojure programming. Be detailed and precise."}]}})) 197 | ;; 4. Stop the chat manually (or just ask to quit) 198 | (flow/stop chat)) 199 | ;; 5. Test the weather function 200 | ;; You: What's the weather like in Paris? 201 | ;; Assistant: I'll check the weather in Paris for you. 202 | ;; [Tool call executed] 203 | ;; The weather in Paris is 17 degrees celsius 204 | ;; 6. CLI usage (alternative to REPL) 205 | ;; From terminal: bin/chat 206 | ;; Or with debug: bin/chat --debug 207 | -------------------------------------------------------------------------------- /TODO.org_archive: -------------------------------------------------------------------------------- 1 | # -*- mode: org -*- 2 | 3 | 4 | Archived entries from file /Users/ovistoica/workspace/simulflow/TODO.org 5 | 6 | 7 | * DONE Add support for configuration change. 8 | CLOSED: [2025-01-28 Tue 09:12] 9 | :PROPERTIES: 10 | :ARCHIVE_TIME: 2025-01-28 Tue 09:12 11 | :ARCHIVE_FILE: ~/workspace/simulflow/TODO.org 12 | :ARCHIVE_CATEGORY: TODO 13 | :ARCHIVE_TODO: DONE 14 | :END: 15 | Usecase: 16 | We have an initial prompt and tools to use. We want to change it based on the custom parameters that are inputted throught the twilio websocket. 17 | Example: On the twilio websocket, we can give custom parameters like script-name, overrides like user name, etc. 18 | 19 | We can use the config-change frame to do this. And every processor takes what it cares about from it. However, you add very specific functionality to the twilio-in transport. So, what you need to do is add a custom-params->config argument. 20 | 21 | #+begin_src clojure 22 | :transport-in {:proc transport/twilio-transport-in 23 | :args {:transport/in-ch in 24 | :twilio/handle-event (fn [event] 25 | {:out {:llm/context ".." 26 | :llm/registered-tools [...]}})} 27 | #+end_src 28 | 29 | * DONE add core.async.flow support 30 | CLOSED: [2025-01-28 Tue 08:51] 31 | :PROPERTIES: 32 | :ARCHIVE_TIME: 2025-01-28 Tue 09:12 33 | :ARCHIVE_FILE: ~/workspace/simulflow/TODO.org 34 | :ARCHIVE_CATEGORY: TODO 35 | :ARCHIVE_TODO: DONE 36 | :END: 37 | :LOGBOOK: 38 | CLOCK: [2025-01-25 Sat 16:35]--[2025-01-25 Sat 17:00] => 0:25 39 | CLOCK: [2025-01-25 Sat 15:18]--[2025-01-25 Sat 15:43] => 0:25 40 | CLOCK: [2025-01-25 Sat 11:14]--[2025-01-25 Sat 11:39] => 0:25 41 | CLOCK: [2025-01-25 Sat 09:50]--[2025-01-25 Sat 10:15] => 0:25 42 | :END: 43 | 44 | * DONE Research a way to add clj-kondo schema type hints for frames in the macro 45 | CLOSED: [2025-01-20 Lun 07:43] 46 | :PROPERTIES: 47 | :ARCHIVE_TIME: 2025-01-28 Tue 09:13 48 | :ARCHIVE_FILE: ~/workspace/simulflow/TODO.org 49 | :ARCHIVE_CATEGORY: TODO 50 | :ARCHIVE_TODO: DONE 51 | :END: 52 | #+begin_src clojure 53 | (defframe my-cool-frame 54 | "This is a cool frame" 55 | {:type :frame.cool/hello 56 | :schema [:map 57 | [:messages LLMContextMessages] 58 | [:tools LLMTools]]}) 59 | #+end_src 60 | 61 | * DONE Add tools calls support :mvp: 62 | CLOSED: [2025-01-28 Tue 09:13] DEADLINE: <2025-01-17 Fri> 63 | :PROPERTIES: 64 | :ARCHIVE_TIME: 2025-01-28 Tue 09:13 65 | :ARCHIVE_FILE: ~/workspace/simulflow/TODO.org 66 | :ARCHIVE_CATEGORY: TODO 67 | :ARCHIVE_TODO: DONE 68 | :END: 69 | :LOGBOOK: 70 | CLOCK: [2025-01-23 Thu 08:23]--[2025-01-23 Thu 08:48] => 0:25 71 | CLOCK: [2025-01-20 Lun 07:26]--[2025-01-20 Lun 07:51] => 0:25 72 | CLOCK: [2025-01-19 Dum 07:56]--[2025-01-19 Dum 08:21] => 0:25 73 | CLOCK: [2025-01-18 Sat 06:36]--[2025-01-18 Sat 06:41] => 0:05 74 | CLOCK: [2025-01-16 Thu 19:19]--[2025-01-16 Thu 19:44] => 0:25 75 | CLOCK: [2025-01-15 Wed 08:53]--[2025-01-15 Wed 09:18] => 0:25 76 | CLOCK: [2025-01-15 Wed 08:16]--[2025-01-15 Wed 08:41] => 0:25 77 | CLOCK: [2025-01-15 Wed 06:30]--[2025-01-15 Wed 06:55] => 0:25 78 | CLOCK: [2025-01-14 Tue 07:09]--[2025-01-14 Tue 07:34] => 0:25 79 | CLOCK: [2025-01-14 Tue 06:25]--[2025-01-14 Tue 06:50] => 0:25 80 | :END: 81 | 82 | * DONE Create buffered output transport that sends chunks of 20ms at a 10ms interval 83 | CLOSED: [2025-01-10 Vin 13:46] 84 | :PROPERTIES: 85 | :ARCHIVE_TIME: 2025-01-28 Tue 09:13 86 | :ARCHIVE_FILE: ~/workspace/simulflow/TODO.org 87 | :ARCHIVE_OLPATH: Add pipeline interruptions 88 | :ARCHIVE_CATEGORY: TODO 89 | :ARCHIVE_TODO: DONE 90 | :ARCHIVE_ITAGS: mvp 91 | :END: 92 | :LOGBOOK: 93 | CLOCK: [2025-01-09 Thu 15:51]--[2025-01-09 Thu 16:16] => 0:25 94 | CLOCK: [2025-01-09 Thu 15:19]--[2025-01-09 Thu 15:44] => 0:25 95 | CLOCK: [2025-01-09 Thu 14:45]--[2025-01-09 Thu 15:10] => 0:25 96 | CLOCK: [2025-01-09 Thu 13:58]--[2025-01-09 Thu 14:23] => 0:25 97 | CLOCK: [2025-01-09 Thu 08:29]--[2025-01-09 Thu 08:54] => 0:25 98 | CLOCK: [2025-01-09 Thu 07:46]--[2025-01-09 Thu 08:11] => 0:25 99 | CLOCK: [2025-01-09 Thu 07:00]--[2025-01-09 Thu 07:25] => 0:25 100 | CLOCK: [2025-01-09 Thu 06:29]--[2025-01-09 Thu 06:54] => 0:25 101 | CLOCK: [2025-01-08 Wed 10:45]--[2025-01-08 Wed 11:10] => 0:25 102 | CLOCK: [2025-01-08 Wed 08:29]--[2025-01-08 Wed 08:54] => 0:25 103 | :END: 104 | 105 | * DONE Handle Start/Stop interruption frames in LLM and TTS and other assemblers 106 | CLOSED: [2025-01-13 Mon 07:53] 107 | :PROPERTIES: 108 | :ARCHIVE_TIME: 2025-01-28 Tue 09:13 109 | :ARCHIVE_FILE: ~/workspace/simulflow/TODO.org 110 | :ARCHIVE_OLPATH: Add pipeline interruptions 111 | :ARCHIVE_CATEGORY: TODO 112 | :ARCHIVE_TODO: DONE 113 | :ARCHIVE_ITAGS: mvp 114 | :END: 115 | :LOGBOOK: 116 | CLOCK: [2025-01-10 Vin 16:29]--[2025-01-10 Vin 16:54] => 0:25 117 | CLOCK: [2025-01-10 Vin 14:15]--[2025-01-10 Vin 14:41] => 0:26 118 | CLOCK: [2025-01-10 Vin 13:46]--[2025-01-10 Vin 14:11] => 0:25 119 | CLOCK: [2025-01-08 Wed 07:01]--[2025-01-08 Wed 07:26] => 0:25 120 | CLOCK: [2025-01-07 Tue 07:17]--[2025-01-07 Tue 07:42] => 0:25 121 | CLOCK: [2025-01-07 Tue 06:20]--[2025-01-07 Tue 06:45] => 0:25 122 | CLOCK: [2025-01-06 Mon 17:07]--[2025-01-06 Mon 17:40] => 0:33 123 | CLOCK: [2025-01-06 Mon 16:36]--[2025-01-06 Mon 17:01] => 0:25 124 | :END: 125 | 126 | * DONE Add assembler that takes in interim transcripts based on VAD 127 | CLOSED: [2025-01-06 Mon 16:35] 128 | :PROPERTIES: 129 | :ARCHIVE_TIME: 2025-01-28 Tue 09:13 130 | :ARCHIVE_FILE: ~/workspace/simulflow/TODO.org 131 | :ARCHIVE_OLPATH: Add pipeline interruptions 132 | :ARCHIVE_CATEGORY: TODO 133 | :ARCHIVE_TODO: DONE 134 | :ARCHIVE_ITAGS: mvp 135 | :END: 136 | :LOGBOOK: 137 | CLOCK: [2025-01-06 Mon 12:28]--[2025-01-06 Mon 12:53] => 0:25 138 | CLOCK: [2025-01-06 Mon 07:37]--[2025-01-06 Mon 08:02] => 0:25 139 | CLOCK: [2025-01-05 Sun 09:21]--[2025-01-05 Sun 09:46] => 0:25 140 | CLOCK: [2025-01-05 Sun 08:18]--[2025-01-05 Sun 08:43] => 0:25 141 | CLOCK: [2025-01-04 Sat 15:22]--[2025-01-04 Sat 15:47] => 0:25 142 | CLOCK: [2025-01-04 Sat 11:04]--[2025-01-04 Sat 11:29] => 0:25 143 | CLOCK: [2025-01-04 Sat 07:14]--[2025-01-04 Sat 07:39] => 0:25 144 | :END: 145 | 146 | * DONE Add VAD events from deepgram 147 | CLOSED: [2025-01-03 Fri 19:41] 148 | :PROPERTIES: 149 | :ARCHIVE_TIME: 2025-01-28 Tue 09:13 150 | :ARCHIVE_FILE: ~/workspace/simulflow/TODO.org 151 | :ARCHIVE_CATEGORY: TODO 152 | :ARCHIVE_TODO: DONE 153 | :END: 154 | :LOGBOOK: 155 | CLOCK: [2025-01-03 Fri 16:25]--[2025-01-03 Fri 16:50] => 0:25 156 | CLOCK: [2025-01-03 Fri 15:36]--[2025-01-03 Fri 16:01] => 0:25 157 | CLOCK: [2025-01-03 Fri 11:01]--[2025-01-03 Fri 11:26] => 0:25 158 | :END: 159 | 160 | * DONE Add schema validation with defaults 161 | CLOSED: [2025-01-03 Fri 11:01] 162 | :PROPERTIES: 163 | :ARCHIVE_TIME: 2025-01-28 Tue 09:13 164 | :ARCHIVE_FILE: ~/workspace/simulflow/TODO.org 165 | :ARCHIVE_CATEGORY: TODO 166 | :ARCHIVE_TODO: DONE 167 | :END: 168 | :LOGBOOK: 169 | CLOCK: [2025-01-03 Fri 07:51]--[2025-01-03 Fri 08:16] => 0:25 170 | CLOCK: [2025-01-03 Fri 07:06]--[2025-01-03 Fri 07:31] => 0:25 171 | CLOCK: [2025-01-03 Fri 06:35]--[2025-01-03 Fri 07:00] => 0:25 172 | :END: 173 | 174 | * DONE Change tool_call declaration to include the handler, to enable changing available tools on the fly 175 | CLOSED: [2025-02-02 Sun 07:31] 176 | :PROPERTIES: 177 | :ARCHIVE_TIME: 2025-02-02 Sun 07:31 178 | :ARCHIVE_FILE: ~/workspace/simulflow/TODO.org 179 | :ARCHIVE_OLPATH: Implement diagram flows into vice-fn 180 | :ARCHIVE_CATEGORY: TODO 181 | :ARCHIVE_TODO: DONE 182 | :END: 183 | If the function :handler returns a channel, the tool-caller will block until a result is put on the channel, optionally with a timeout 184 | #+begin_src clojure 185 | :functions [{:type :function 186 | :function 187 | {:name "record_party_size" 188 | :handler (fn [{:keys [size]}] ...) 189 | :description "Record the number of people in the party" 190 | :parameters 191 | {:type :object 192 | :properties 193 | {:size {:type :integer 194 | :minimum 1 195 | :maximum 12}} 196 | :required [:size]} 197 | :transition-to :get-time}}] 198 | #+end_src 199 | 200 | After this, basically we can just emit a =frame/llm-context= and that will update the current context. However the scenario manager needs to 201 | 202 | * DONE Fix end the call function 203 | CLOSED: [2025-02-05 Wed 08:34] 204 | :PROPERTIES: 205 | :ARCHIVE_TIME: 2025-02-05 Wed 08:34 206 | :ARCHIVE_FILE: ~/workspace/simulflow/TODO.org 207 | :ARCHIVE_CATEGORY: TODO 208 | :ARCHIVE_TODO: DONE 209 | :END: 210 | :LOGBOOK: 211 | CLOCK: [2025-01-29 Wed 18:03]--[2025-01-29 Wed 18:10] => 0:07 212 | :END: 213 | 214 | * Differences between pipecat and simulflow 215 | :PROPERTIES: 216 | :ARCHIVE_TIME: 2025-08-23 Sat 09:58 217 | :ARCHIVE_FILE: ~/workspace/simulflow/TODO.org 218 | :ARCHIVE_OLPATH: Add pipeline interruptions 219 | :ARCHIVE_CATEGORY: TODO 220 | :ARCHIVE_ITAGS: mvp 221 | :END: 222 | 1. (I think) simulflow TTS processors whould keep a =:pipeline/interrupted?= 223 | state because when the processor receives a =speak-frame=, it sends it on the 224 | websocket connection to the actual TTS provider that may send one or more 225 | events back that need to be accumulated to construct the full audio 226 | eequivalent of the text from the =speak-frame=. Therefore we keep the 227 | =pipeline/interrupted?= flag so when new data is received on the websocket 228 | the processor drops them. 229 | 2. We need a way to clear the "playback queue". Currently the playback queue is 230 | represented by the [[file:src/simulflow/transport/out.clj::audio-write-ch (a/chan 1024)\]][audio-write-channel]] defined. There is a [[file:src/simulflow/async.clj::(defn drain-channel!][drain-channel!]] 231 | function which will work but we need to introduce two channels to communicate 232 | with the [[file:src/simulflow/transport/out.clj::(vthread-loop \[\]][process running in a vthread]] that sends audio to out. One for 233 | commands to drain audio, and one on which to take audio from (the current 234 | existing one) 235 | 3. Pipecat uses a bidirectional queue system between processors: 236 | Transport in <-> Transcriptor <-> Context Aggregator <-> LLM <-> TTS <-> a 237 | -------------------------------------------------------------------------------- /src/simulflow/processors/elevenlabs.clj: -------------------------------------------------------------------------------- 1 | (ns simulflow.processors.elevenlabs 2 | (:require 3 | [clojure.core.async :as a] 4 | [clojure.core.async.flow :as flow] 5 | [hato.websocket :as ws] 6 | [simulflow.async :refer [vthread-loop]] 7 | [simulflow.frame :as frame] 8 | [simulflow.schema :as schema] 9 | [simulflow.utils.core :as u] 10 | [taoensso.telemere :as t]) 11 | (:import 12 | (java.nio HeapCharBuffer))) 13 | 14 | (def ^:private xi-tts-websocket-url "wss://api.elevenlabs.io/v1/text-to-speech/%s/stream-input") 15 | 16 | (def elevenlabs-encoding 17 | "Mapping from clojure sound encoding to elevenlabs format" 18 | {:ulaw :ulaw 19 | :mp3 :mp3 20 | :pcm-signed :pcm 21 | :pcm-unsigned :pcm 22 | :pcm-float :pcm}) 23 | 24 | (defn encoding->elevenlabs 25 | [format sample-rate] 26 | (keyword (str (name (elevenlabs-encoding format)) "_" sample-rate))) 27 | 28 | (defn make-elevenlabs-ws-url 29 | [args] 30 | (let [{:audio.out/keys [encoding sample-rate] 31 | :pipeline/keys [language] 32 | :elevenlabs/keys [model-id voice-id] 33 | :or {model-id "eleven_flash_v2_5" 34 | sample-rate 24000}} 35 | args] 36 | (assert voice-id "Voice ID is required") 37 | (u/append-search-params (format xi-tts-websocket-url voice-id) 38 | {:model_id model-id 39 | :language_code language 40 | :output_format (encoding->elevenlabs encoding sample-rate)}))) 41 | 42 | (defn begin-stream-message 43 | [{:voice/keys [stability similarity-boost use-speaker-boost?] 44 | :elevenlabs/keys [api-key] 45 | :or {stability 0.5 46 | similarity-boost 0.8 47 | use-speaker-boost? true}}] 48 | (u/json-str {:text " " 49 | :voice_settings {:stability stability 50 | :similarity_boost similarity-boost 51 | :use_speaker_boost use-speaker-boost?} 52 | :xi_api_key api-key})) 53 | 54 | (def close-stream-message 55 | (u/json-str {:text ""})) 56 | 57 | (def keep-alive-message 58 | "Sent to keep the connection alive" 59 | (u/json-str {:text " "})) 60 | 61 | (defn text-message 62 | [text] 63 | (u/json-str {:text (str text " ") 64 | :flush true})) 65 | 66 | (def ElevenLabsTTSConfig 67 | "Configuration for Elevenlabs TextToSpeech service" 68 | [:map 69 | [:elevenlabs/api-key 70 | [:string 71 | {:min 32 ;; ElevenLabs API keys are typically long 72 | :secret true ;; Marks this as sensitive data 73 | :description "ElevenLabs API key"}]] 74 | [:pipeline/language {:default :en} schema/Language] 75 | [:elevenlabs/model-id {:default :eleven_flash_v2_5 76 | :description "ElevenLabs model identifier"} 77 | (schema/flex-enum 78 | [:eleven_multilingual_v2 :eleven_turbo_v2_5 :eleven_turbo_v2 :eleven_monolingual_v1 79 | :eleven_multilingual_v1 :eleven_multilingual_sts_v2 :eleven_flash_v2 :eleven_flash_v2_5 :eleven_english_sts_v2])] 80 | [:elevenlabs/voice-id 81 | [:string 82 | {:min 20 ;; ElevenLabs voice IDs are fixed length 83 | :max 20 84 | :description "ElevenLabs voice identifier"}]] 85 | [:voice/stability {:default 0.5 86 | :description "Voice stability factor (0.0 to 1.0)"} 87 | [:double 88 | {:min 0.0 89 | :max 1.0}]] 90 | [:voice/similarity-boost {:default 0.8 91 | :description "Voice similarity boost factor (0.0 to 1.0)"} 92 | [:double 93 | {:min 0.0 94 | :max 1.0}]] 95 | [:voice/use-speaker-boost? {:default true 96 | :description "Whether to enable speaker boost enhancement"} 97 | :boolean] 98 | [:audio.out/encoding {:default :pcm-signed 99 | :description "The encoding for the generated audio. By default uses PCM but can be changed for others for example ulaw to use in telephony context"} schema/AudioEncoding] 100 | [:audio.out/sample-rate {:default 24000 101 | :description "The sample rate at which elevenlabs will generate audio"} 102 | [:enum 8000 16000 22050 24000 44100]]]) 103 | 104 | (defn accumulate-json-response 105 | "Pure function to accumulate JSON response fragments. 106 | Returns [new-accumulator parsed-json-or-nil]" 107 | [current-accumulator new-fragment] 108 | (let [combined (str current-accumulator new-fragment) 109 | parsed (u/parse-if-json combined)] 110 | (if (map? parsed) 111 | ["" parsed] ; Successfully parsed, reset accumulator 112 | [combined nil]))) ; Still accumulating 113 | 114 | (defn process-speak-frame 115 | "Pure function to process speak frame into WebSocket message. 116 | Returns WebSocket message string." 117 | [speak-frame] 118 | (text-message (:frame/data speak-frame))) 119 | 120 | (defn process-websocket-message 121 | "Pure function to process WebSocket message and update accumulator state. 122 | Returns [new-state output-map]" 123 | [state message timestamp] 124 | (let [current-accumulator (::accumulator state) 125 | [new-accumulator parsed-json] (accumulate-json-response current-accumulator message)] 126 | (if parsed-json 127 | ;; JSON parsing complete 128 | (let [new-state (assoc state ::accumulator new-accumulator) 129 | frames (when-let [audio (:audio parsed-json)] 130 | [(frame/audio-output-raw {:audio (u/decode-base64 audio) 131 | :sample-rate (:audio.out/sample-rate state)} 132 | {:timestamp timestamp}) 133 | (frame/xi-audio-out parsed-json {:timestamp timestamp})])] 134 | [new-state (apply frame/send frames)]) 135 | ;; Still accumulating 136 | [(assoc state ::accumulator new-accumulator) {}]))) 137 | 138 | ;; ============================================================================ 139 | ;; WebSocket Configuration 140 | ;; ============================================================================ 141 | 142 | (defn create-websocket-config 143 | "Pure function to create WebSocket configuration map" 144 | [args ws-read ws-write] 145 | (let [configuration-msg (begin-stream-message args)] 146 | {:on-open (fn [ws] 147 | (t/log! {:level :debug :id :elevenlabs} ["Websocket connection open. Sending configuration message" configuration-msg]) 148 | (ws/send! ws configuration-msg)) 149 | :on-message (fn [_ws ^HeapCharBuffer data _last?] 150 | (a/put! ws-read (str data))) 151 | :on-error (fn [_ e] 152 | (t/log! {:level :error :id :elevenlabs} ["Websocket error" (ex-message e)])) 153 | :on-close (fn [_ws code reason] 154 | (t/log! {:level :debug :id :elevenlabs} ["Websocket connection closed" "Code:" code "Reason:" reason]))})) 155 | 156 | (def elevenlabs-tts-describe 157 | {:ins {:sys-in "Channel for system messages that take priority" 158 | :in "Channel for audio input frames (from transport-in) "} 159 | :outs {:sys-out "Channel for system messages that have priority" 160 | :out "Channel on which transcription frames are put"} 161 | :params (schema/->describe-parameters ElevenLabsTTSConfig) 162 | :workload :io}) 163 | 164 | (defn elevenlabs-tts-init! [params] 165 | (let [args (schema/parse-with-defaults ElevenLabsTTSConfig params) 166 | url (make-elevenlabs-ws-url args) 167 | ws-read (a/chan 100) 168 | ws-write (a/chan 100) 169 | alive? (atom true) 170 | conf (assoc (create-websocket-config args ws-read ws-write) 171 | :on-close (fn [_ws code reason] 172 | (reset! alive? false) 173 | (t/log! {:level :debug :id :elevenlabs} ["Websocket connection closed" "Code:" code "Reason:" reason]))) 174 | _ (t/log! {:level :debug :id :elevenlabs :data url} "Connecting to TTS websocket") 175 | ws-conn @(ws/websocket url conf)] 176 | (vthread-loop [] 177 | (when @alive? 178 | (when-let [msg (a/ :cat [:any] :any]]]]) 27 | 28 | (def ScenarioConfig 29 | [:and [:map {:closed true} 30 | [:initial-node :keyword] 31 | [:nodes [:map-of 32 | :keyword 33 | [:map {:closed true} 34 | [:run-llm? {:optional true} :boolean] 35 | [:role-messages {:optional true} [:vector schema/LLMSystemMessage]] 36 | [:task-messages [:vector schema/LLMSystemMessage]] 37 | [:functions [:vector [:or 38 | schema/LLMFunctionToolDefinitionWithHandling 39 | schema/LLMTransitionToolDefinition]]] 40 | [:pre-actions {:optional true 41 | :description "Actions to be invoked when the node is selected."} [:vector ScenarioAction]] 42 | [:post-actions {:optional true 43 | :description "Actions to be invoked when the node will be replaced."} [:vector ScenarioAction]]]]]] 44 | [:fn {:error/message "Initial node not defined"} 45 | (fn [sc] 46 | (boolean (get-in sc [:nodes (:initial-node sc)])))] 47 | [:fn {:error/fn (fn [{:keys [value]} _] 48 | (let [nodes (set (keys (:nodes value))) 49 | transitions (->> value 50 | :nodes 51 | vals 52 | (mapcat :functions) 53 | (keep (fn [f] (get-in f [:function :transition-to]))) 54 | (remove #(or (nil? %) 55 | (fn? %)))) 56 | invalid-transition (first (remove nodes transitions))] 57 | (when invalid-transition 58 | (format "Unreachable node: %s" invalid-transition))))} 59 | (fn [{:keys [nodes]}] 60 | (let [defined-nodes (set (keys nodes)) 61 | transitions (->> nodes 62 | vals 63 | (mapcat :functions) 64 | (keep (fn [f] (get-in f [:function :transition-to]))) 65 | (remove #(or (nil? %) 66 | (fn? %))))] 67 | (every? defined-nodes transitions)))]]) 68 | 69 | (defprotocol Scenario 70 | (start [s] "Start the scenario") 71 | (set-node [s node] "Moves to the current node of the conversation") 72 | (current-node [s] "Get current node")) 73 | 74 | (defn transition-fn 75 | "Transform a function declaration into a transition function. A transition 76 | function calls the original function handler, and then transitions the 77 | scenario to the :transition-to node from f 78 | 79 | scenario - scenario that will be transitioned 80 | tool - transition tool declaration. See `schema/LLMTransitionToolDefinition` 81 | " 82 | [scenario tool] 83 | (let [fndef (:function tool) 84 | handler (:handler fndef) 85 | 86 | next-node-or-fn (:transition-to fndef) 87 | cb (when next-node-or-fn 88 | (if (fn? next-node-or-fn) 89 | (fn [args] 90 | (let [next-node (next-node-or-fn args)] 91 | (set-node scenario next-node))) 92 | (fn [_] (set-node scenario next-node-or-fn))))] 93 | (cond-> tool 94 | true (update-in [:function] dissoc :transition-to) 95 | cb (assoc-in [:function :transition-cb] cb) 96 | (nil? handler) (assoc-in [:function :handler] (fn [_] {:status :success}))))) 97 | 98 | (defn scenario-manager 99 | [{:keys [scenario-config flow flow-in-coord]}] 100 | (when-let [errors (me/humanize (m/explain ScenarioConfig scenario-config))] 101 | (throw (ex-info "Invalid scenario config" {:errors errors}))) 102 | 103 | (let [current-node (atom nil) 104 | nodes (:nodes scenario-config) 105 | initialized? (atom false) 106 | tts-action? #(contains? #{:tts-say "tts-say"} (:type %)) 107 | end-action? #(contains? #{:end-conversation "end-conversation"} (:type %)) 108 | handle-action (fn [a] 109 | (cond 110 | (tts-action? a) 111 | (flow/inject flow flow-in-coord (if (coll? (:text a)) 112 | (mapv #(frame/speak-frame %) (:text a)) 113 | [(frame/speak-frame (:text a))])) 114 | (end-action? a) (flow/stop flow) 115 | :else ((:handler a))))] 116 | (reify Scenario 117 | (current-node [_] @current-node) 118 | (set-node [this node-id] 119 | (assert (get-in scenario-config [:nodes node-id]) (str "Invalid node: " node-id)) 120 | (t/log! :info ["SCENARIO" "NEW NODE" node-id]) 121 | (let [node (get nodes node-id) 122 | tools (mapv (partial transition-fn this) (:functions node)) 123 | context (vec (->> (concat (:role-messages node) (:task-messages node)) 124 | (remove nil?))) 125 | ;; post actions from previous node 126 | post-actions (when @current-node (get-in nodes [@current-node :post-actions])) 127 | pre-actions (:pre-actions node)] 128 | 129 | (try 130 | (when (seq post-actions) (doseq [a post-actions] 131 | (handle-action a))) 132 | 133 | (reset! current-node node-id) 134 | (flow/inject 135 | flow 136 | flow-in-coord 137 | [(frame/scenario-context-update {:messages context 138 | :tools tools 139 | :properties {:run-llm? (get node :run-llm? true)}})]) 140 | (when (seq pre-actions) (doseq [a pre-actions] 141 | (handle-action a))) 142 | 143 | (catch Exception e 144 | (t/log! :error e))))) 145 | 146 | (start [s] 147 | (when-not @initialized? 148 | (reset! initialized? true) 149 | (set-node s (:initial-node scenario-config))))))) 150 | 151 | (defn scenario-in-fn 152 | "Process that acts as a input for the scenario manager into the flow. This 153 | process will direct specific frames to specific outs. Example: speak-frame 154 | will be directed to :speak-out channel (should be connected to a text to 155 | speech process)" 156 | ([] {:ins {:scenario-in "Channel on which the scenario will put frames."} 157 | :outs {:sys-out "Channel where system frames will be put (follows app convention)"}}) 158 | ([_] nil) 159 | ([state _] state) 160 | ([_ in frame] 161 | (t/log! {:id :scenario-in :data frame :msg "GOT NEW FRAME" :level :debug}) 162 | (when (= in :scenario-in) 163 | [nil (frame/send frame)]))) 164 | 165 | (def scenario-in-process (flow/process scenario-in-fn)) 166 | 167 | (comment 168 | (scenario-manager 169 | {:flow (flow/create-flow {:procs {} 170 | :conns []}) 171 | :scenario 172 | {:initial-node :start 173 | :nodes 174 | {:start 175 | {:role-messages [{:role :system 176 | :content "You are a restaurant reservation assistant for La Maison, an upscale French restaurant. You must ALWAYS use one of the available functions to progress the conversation. This is a phone conversations and your responses will be converted to audio. Avoid outputting special characters and emojis. Be casual and friendly."}] 177 | :task-messages [{:role :system 178 | :content "Warmly greet the customer and ask how many people are in their party."}] 179 | :functions [{:type :function 180 | :function 181 | {:name "record_party_size" 182 | :handler (fn [{:keys [size]}] size) 183 | :description "Record the number of people in the party" 184 | :parameters 185 | {:type :object 186 | :properties 187 | {:size {:type :integer 188 | :minimum 1 189 | :maximum 12}} 190 | :required [:size]} 191 | :transition-to :get-time}}]} 192 | :get-time 193 | {:task-messages [{:role :system 194 | :content "Ask what time they'd like to dine. Restaurant is open 5 PM to 10 PM. After they provide a time, confirm it's within operating hours before recording. Use 24-hour format for internal recording (e.g., 17:00 for 5 PM)."}] 195 | :functions [{:type :function 196 | :function {:name "record_time" 197 | :handler (fn [{:keys [time]}] time) 198 | :description "Record the requested time" 199 | :parameters {:type :object 200 | :properties {:time {:type :string 201 | :pattern "^(17|18|19|20|21|22):([0-5][0-9])$" 202 | :description "Reservation time in 24-hour format (17:00-22:00)"}} 203 | :required [:time]} 204 | :transition_to "confirm"}}]}}}})) 205 | 206 | (comment 207 | (me/humanize (m/explain schema/LLMTransitionToolDefinition {:type :function 208 | :function 209 | {:name "record_party_size" 210 | :handler (fn [] :1) 211 | :description "Record the number of people in the party" 212 | :parameters 213 | {:type :object 214 | :properties 215 | {:size {:type :integer 216 | :min 1 217 | :max 12 218 | :description "The people that want to dine"}} 219 | :required [:size]} 220 | :transition-to :get-time}}))) 221 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC 2 | LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM 3 | CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. 4 | 5 | 1. DEFINITIONS 6 | 7 | "Contribution" means: 8 | 9 | a) in the case of the initial Contributor, the initial code and 10 | documentation distributed under this Agreement, and 11 | 12 | b) in the case of each subsequent Contributor: 13 | 14 | i) changes to the Program, and 15 | 16 | ii) additions to the Program; 17 | 18 | where such changes and/or additions to the Program originate from and are 19 | distributed by that particular Contributor. A Contribution 'originates' from 20 | a Contributor if it was added to the Program by such Contributor itself or 21 | anyone acting on such Contributor's behalf. Contributions do not include 22 | additions to the Program which: (i) are separate modules of software 23 | distributed in conjunction with the Program under their own license 24 | agreement, and (ii) are not derivative works of the Program. 25 | 26 | "Contributor" means any person or entity that distributes the Program. 27 | 28 | "Licensed Patents" mean patent claims licensable by a Contributor which are 29 | necessarily infringed by the use or sale of its Contribution alone or when 30 | combined with the Program. 31 | 32 | "Program" means the Contributions distributed in accordance with this 33 | Agreement. 34 | 35 | "Recipient" means anyone who receives the Program under this Agreement, 36 | including all Contributors. 37 | 38 | 2. GRANT OF RIGHTS 39 | 40 | a) Subject to the terms of this Agreement, each Contributor hereby grants 41 | Recipient a non-exclusive, worldwide, royalty-free copyright license to 42 | reproduce, prepare derivative works of, publicly display, publicly perform, 43 | distribute and sublicense the Contribution of such Contributor, if any, and 44 | such derivative works, in source code and object code form. 45 | 46 | b) Subject to the terms of this Agreement, each Contributor hereby grants 47 | Recipient a non-exclusive, worldwide, royalty-free patent license under 48 | Licensed Patents to make, use, sell, offer to sell, import and otherwise 49 | transfer the Contribution of such Contributor, if any, in source code and 50 | object code form. This patent license shall apply to the combination of the 51 | Contribution and the Program if, at the time the Contribution is added by the 52 | Contributor, such addition of the Contribution causes such combination to be 53 | covered by the Licensed Patents. The patent license shall not apply to any 54 | other combinations which include the Contribution. No hardware per se is 55 | licensed hereunder. 56 | 57 | c) Recipient understands that although each Contributor grants the licenses 58 | to its Contributions set forth herein, no assurances are provided by any 59 | Contributor that the Program does not infringe the patent or other 60 | intellectual property rights of any other entity. Each Contributor disclaims 61 | any liability to Recipient for claims brought by any other entity based on 62 | infringement of intellectual property rights or otherwise. As a condition to 63 | exercising the rights and licenses granted hereunder, each Recipient hereby 64 | assumes sole responsibility to secure any other intellectual property rights 65 | needed, if any. For example, if a third party patent license is required to 66 | allow Recipient to distribute the Program, it is Recipient's responsibility 67 | to acquire that license before distributing the Program. 68 | 69 | d) Each Contributor represents that to its knowledge it has sufficient 70 | copyright rights in its Contribution, if any, to grant the copyright license 71 | set forth in this Agreement. 72 | 73 | 3. REQUIREMENTS 74 | 75 | A Contributor may choose to distribute the Program in object code form under 76 | its own license agreement, provided that: 77 | 78 | a) it complies with the terms and conditions of this Agreement; and 79 | 80 | b) its license agreement: 81 | 82 | i) effectively disclaims on behalf of all Contributors all warranties and 83 | conditions, express and implied, including warranties or conditions of title 84 | and non-infringement, and implied warranties or conditions of merchantability 85 | and fitness for a particular purpose; 86 | 87 | ii) effectively excludes on behalf of all Contributors all liability for 88 | damages, including direct, indirect, special, incidental and consequential 89 | damages, such as lost profits; 90 | 91 | iii) states that any provisions which differ from this Agreement are offered 92 | by that Contributor alone and not by any other party; and 93 | 94 | iv) states that source code for the Program is available from such 95 | Contributor, and informs licensees how to obtain it in a reasonable manner on 96 | or through a medium customarily used for software exchange. 97 | 98 | When the Program is made available in source code form: 99 | 100 | a) it must be made available under this Agreement; and 101 | 102 | b) a copy of this Agreement must be included with each copy of the Program. 103 | 104 | Contributors may not remove or alter any copyright notices contained within 105 | the Program. 106 | 107 | Each Contributor must identify itself as the originator of its Contribution, 108 | if any, in a manner that reasonably allows subsequent Recipients to identify 109 | the originator of the Contribution. 110 | 111 | 4. COMMERCIAL DISTRIBUTION 112 | 113 | Commercial distributors of software may accept certain responsibilities with 114 | respect to end users, business partners and the like. While this license is 115 | intended to facilitate the commercial use of the Program, the Contributor who 116 | includes the Program in a commercial product offering should do so in a 117 | manner which does not create potential liability for other Contributors. 118 | Therefore, if a Contributor includes the Program in a commercial product 119 | offering, such Contributor ("Commercial Contributor") hereby agrees to defend 120 | and indemnify every other Contributor ("Indemnified Contributor") against any 121 | losses, damages and costs (collectively "Losses") arising from claims, 122 | lawsuits and other legal actions brought by a third party against the 123 | Indemnified Contributor to the extent caused by the acts or omissions of such 124 | Commercial Contributor in connection with its distribution of the Program in 125 | a commercial product offering. The obligations in this section do not apply 126 | to any claims or Losses relating to any actual or alleged intellectual 127 | property infringement. In order to qualify, an Indemnified Contributor must: 128 | a) promptly notify the Commercial Contributor in writing of such claim, and 129 | b) allow the Commercial Contributor to control, and cooperate with the 130 | Commercial Contributor in, the defense and any related settlement 131 | negotiations. The Indemnified Contributor may participate in any such claim 132 | at its own expense. 133 | 134 | For example, a Contributor might include the Program in a commercial product 135 | offering, Product X. That Contributor is then a Commercial Contributor. If 136 | that Commercial Contributor then makes performance claims, or offers 137 | warranties related to Product X, those performance claims and warranties are 138 | such Commercial Contributor's responsibility alone. Under this section, the 139 | Commercial Contributor would have to defend claims against the other 140 | Contributors related to those performance claims and warranties, and if a 141 | court requires any other Contributor to pay any damages as a result, the 142 | Commercial Contributor must pay those damages. 143 | 144 | 5. NO WARRANTY 145 | 146 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON 147 | AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 148 | EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR 149 | CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A 150 | PARTICULAR PURPOSE. Each Recipient is solely responsible for determining the 151 | appropriateness of using and distributing the Program and assumes all risks 152 | associated with its exercise of rights under this Agreement , including but 153 | not limited to the risks and costs of program errors, compliance with 154 | applicable laws, damage to or loss of data, programs or equipment, and 155 | unavailability or interruption of operations. 156 | 157 | 6. DISCLAIMER OF LIABILITY 158 | 159 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY 160 | CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, 161 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION 162 | LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 163 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 164 | ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE 165 | EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY 166 | OF SUCH DAMAGES. 167 | 168 | 7. GENERAL 169 | 170 | If any provision of this Agreement is invalid or unenforceable under 171 | applicable law, it shall not affect the validity or enforceability of the 172 | remainder of the terms of this Agreement, and without further action by the 173 | parties hereto, such provision shall be reformed to the minimum extent 174 | necessary to make such provision valid and enforceable. 175 | 176 | If Recipient institutes patent litigation against any entity (including a 177 | cross-claim or counterclaim in a lawsuit) alleging that the Program itself 178 | (excluding combinations of the Program with other software or hardware) 179 | infringes such Recipient's patent(s), then such Recipient's rights granted 180 | under Section 2(b) shall terminate as of the date such litigation is filed. 181 | 182 | All Recipient's rights under this Agreement shall terminate if it fails to 183 | comply with any of the material terms or conditions of this Agreement and 184 | does not cure such failure in a reasonable period of time after becoming 185 | aware of such noncompliance. If all Recipient's rights under this Agreement 186 | terminate, Recipient agrees to cease use and distribution of the Program as 187 | soon as reasonably practicable. However, Recipient's obligations under this 188 | Agreement and any licenses granted by Recipient relating to the Program shall 189 | continue and survive. 190 | 191 | Everyone is permitted to copy and distribute copies of this Agreement, but in 192 | order to avoid inconsistency the Agreement is copyrighted and may only be 193 | modified in the following manner. The Agreement Steward reserves the right to 194 | publish new versions (including revisions) of this Agreement from time to 195 | time. No one other than the Agreement Steward has the right to modify this 196 | Agreement. The Eclipse Foundation is the initial Agreement Steward. The 197 | Eclipse Foundation may assign the responsibility to serve as the Agreement 198 | Steward to a suitable separate entity. Each new version of the Agreement will 199 | be given a distinguishing version number. The Program (including 200 | Contributions) may always be distributed subject to the version of the 201 | Agreement under which it was received. In addition, after a new version of 202 | the Agreement is published, Contributor may elect to distribute the Program 203 | (including its Contributions) under the new version. Except as expressly 204 | stated in Sections 2(a) and 2(b) above, Recipient receives no rights or 205 | licenses to the intellectual property of any Contributor under this 206 | Agreement, whether expressly, by implication, estoppel or otherwise. All 207 | rights in the Program not expressly granted under this Agreement are 208 | reserved. 209 | 210 | This Agreement is governed by the laws of the State of New York and the 211 | intellectual property laws of the United States of America. No party to this 212 | Agreement will bring a legal action under this Agreement more than one year 213 | after the cause of action arose. Each party waives its rights to a jury trial 214 | in any resulting litigation. 215 | --------------------------------------------------------------------------------