├── mise.development.toml ├── .yarnrc.yml ├── .dockerignore ├── .yarn └── install-state.gz ├── resources ├── public │ └── chat_app │ │ ├── favicon.ico │ │ ├── kudos-logo.png │ │ ├── ai-guide-logo.jpg │ │ ├── altinn-assistant-logo.png │ │ ├── icons │ │ ├── old │ │ │ ├── down-arrow.svg │ │ │ ├── tick.svg │ │ │ ├── right-arrow.svg │ │ │ ├── x.svg │ │ │ ├── new-chat.svg │ │ │ ├── plus.svg │ │ │ ├── send.svg │ │ │ ├── search.svg │ │ │ ├── no-data.svg │ │ │ ├── edit.svg │ │ │ └── delete.svg │ │ ├── circle-stop.svg │ │ ├── unchecked_checkbox.svg │ │ ├── ellipsis.svg │ │ ├── panel-left-close.svg │ │ ├── panel-left-open.svg │ │ ├── copy.svg │ │ ├── pencil.svg │ │ ├── refresh-cw.svg │ │ ├── loader.svg │ │ ├── square-pen.svg │ │ ├── speech.svg │ │ ├── checked_checkbox.svg │ │ └── progress-circle.svg │ │ ├── bot.svg │ │ ├── no-data.svg │ │ ├── index.html │ │ └── digdir_icon.svg └── input.css ├── glossary.txt ├── renovate.json ├── tailwind.config.js ├── fly-staging-ai-guide.toml ├── src-prod ├── logback.xml └── prod.cljc ├── config ├── ai_guide_staging.edn ├── ai_guide_dev.edn ├── ka_dev.edn ├── ka_staging.edn ├── ka_test.edn ├── ka_sandbox.edn ├── ka_prod.edn └── ka_next.edn ├── fly-next-kunnskap.toml ├── fly-sandbox-kunnskap.toml ├── fly-prod-kunnskap.toml ├── fly-test-kunnskap.toml ├── fly-staging-kunnskap.toml ├── .gitignore ├── src-dev ├── logback.xml ├── dev.cljc └── user.clj ├── src ├── chat_app │ ├── kit.cljc │ ├── rhizome.cljc │ ├── main.cljc │ ├── config_ui.cljc │ ├── debug.cljc │ ├── filters.cljc │ ├── ui.cljc │ ├── entities.cljc │ ├── auth_ui.cljc │ ├── auth.clj │ ├── chat.cljc │ ├── rag_test.cljc │ ├── server_jetty.clj │ └── webauthn.cljc ├── services │ ├── openai.cljc │ └── system.cljc └── models │ └── db.cljc ├── package.json ├── Dockerfile ├── shadow-cljs.edn ├── LICENSE ├── mise.toml ├── README.md ├── src-build └── build.clj └── deps.edn /mise.development.toml: -------------------------------------------------------------------------------- 1 | [tools] 2 | -------------------------------------------------------------------------------- /.yarnrc.yml: -------------------------------------------------------------------------------- 1 | nodeLinker: node-modules 2 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | fly*.toml 2 | Dockerfile 3 | .dockerignore 4 | node_modules 5 | .git 6 | -------------------------------------------------------------------------------- /.yarn/install-state.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/altinn/digdir-rag/main/.yarn/install-state.gz -------------------------------------------------------------------------------- /resources/public/chat_app/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/altinn/digdir-rag/main/resources/public/chat_app/favicon.ico -------------------------------------------------------------------------------- /glossary.txt: -------------------------------------------------------------------------------- 1 | message - An item which takes up vertical space in a conversation, e.g 2 | a user message or a filter. 3 | -------------------------------------------------------------------------------- /resources/public/chat_app/kudos-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/altinn/digdir-rag/main/resources/public/chat_app/kudos-logo.png -------------------------------------------------------------------------------- /resources/public/chat_app/ai-guide-logo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/altinn/digdir-rag/main/resources/public/chat_app/ai-guide-logo.jpg -------------------------------------------------------------------------------- /resources/public/chat_app/altinn-assistant-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/altinn/digdir-rag/main/resources/public/chat_app/altinn-assistant-logo.png -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": [ 4 | "local>Altinn/renovate-config" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /resources/public/chat_app/icons/old/down-arrow.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /resources/public/chat_app/icons/old/tick.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /resources/public/chat_app/icons/old/right-arrow.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /resources/public/chat_app/icons/old/x.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /resources/public/chat_app/icons/old/new-chat.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /resources/public/chat_app/icons/old/plus.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /resources/public/chat_app/icons/circle-stop.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /resources/public/chat_app/icons/unchecked_checkbox.svg: -------------------------------------------------------------------------------- 1 | 2 | 13 | -------------------------------------------------------------------------------- /tailwind.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | content: ["./src/**/*", "./resources/public/index.html"], 3 | // darkMode: 'selector', 4 | theme: { 5 | extend: {}, 6 | }, 7 | variants: { 8 | extend: { 9 | visibility: ['group-hover'], 10 | }, 11 | }, 12 | // plugins: [require('@tailwindcss/typography')], 13 | }; 14 | -------------------------------------------------------------------------------- /resources/public/chat_app/icons/ellipsis.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /resources/public/chat_app/icons/panel-left-close.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /resources/public/chat_app/icons/panel-left-open.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /resources/public/chat_app/icons/copy.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /resources/public/chat_app/bot.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /resources/public/chat_app/icons/old/send.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /resources/public/chat_app/icons/old/search.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /resources/public/chat_app/icons/pencil.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /resources/public/chat_app/icons/refresh-cw.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /resources/public/chat_app/no-data.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /fly-staging-ai-guide.toml: -------------------------------------------------------------------------------- 1 | 2 | app = 'staging-ai-guide' 3 | primary_region = 'arn' 4 | 5 | [build] 6 | 7 | [env] 8 | PORT = '8080' 9 | 10 | [http_service] 11 | internal_port = 8080 12 | force_https = true 13 | auto_stop_machines = 'suspend' 14 | auto_start_machines = true 15 | min_machines_running = 1 16 | processes = ['app'] 17 | 18 | [[vm]] 19 | memory = '1gb' 20 | cpu_kind = 'shared' 21 | cpus = 1 22 | -------------------------------------------------------------------------------- /src-prod/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | %highlight(%-5level) %logger: %msg%n 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /resources/public/chat_app/icons/loader.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /resources/public/chat_app/icons/old/no-data.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /resources/public/chat_app/icons/old/edit.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /resources/public/chat_app/icons/square-pen.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /resources/public/chat_app/icons/old/delete.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /resources/public/chat_app/icons/speech.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /resources/public/chat_app/icons/checked_checkbox.svg: -------------------------------------------------------------------------------- 1 | 2 | 13 | 21 | -------------------------------------------------------------------------------- /config/ai_guide_staging.edn: -------------------------------------------------------------------------------- 1 | {:all-entities-image "bot.svg" 2 | :entities [{:id "7i8dadbe-0101-f0e1-92b8-40b10a61cdcd" 3 | :name "KI-veileder - staging" 4 | :image "ai-guide-logo.jpg" 5 | :docs-collection "AI-GUIDE_docs_2024-10-28" 6 | :chunks-collection "AI-GUIDE_chunks_2024-10-28" 7 | :phrases-collection "AI-GUIDE_phrases_2024-10-28" 8 | :phrase-gen-prompt "keyword-search" 9 | :reasoning-languages ["en" "no"] 10 | :prompt ""}]} -------------------------------------------------------------------------------- /resources/public/chat_app/icons/progress-circle.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /fly-next-kunnskap.toml: -------------------------------------------------------------------------------- 1 | # fly.toml app configuration file generated for next-kunnskap on 2024-12-01T16:13:13+01:00 2 | # 3 | # See https://fly.io/docs/reference/configuration/ for information about how to use this file. 4 | # 5 | 6 | app = 'next-kunnskap' 7 | primary_region = 'arn' 8 | 9 | [build] 10 | 11 | [env] 12 | PORT = '8080' 13 | 14 | [http_service] 15 | internal_port = 8080 16 | force_https = true 17 | auto_stop_machines = 'stop' 18 | auto_start_machines = true 19 | min_machines_running = 1 20 | processes = ['app'] 21 | 22 | [[vm]] 23 | memory = '2gb' 24 | cpu_kind = 'performance' 25 | cpus = 1 26 | -------------------------------------------------------------------------------- /fly-sandbox-kunnskap.toml: -------------------------------------------------------------------------------- 1 | # fly.toml app configuration file generated for next-kunnskap on 2024-12-01T16:13:13+01:00 2 | # 3 | # See https://fly.io/docs/reference/configuration/ for information about how to use this file. 4 | # 5 | 6 | app = 'sandbox-kunnskap' 7 | primary_region = 'arn' 8 | 9 | [build] 10 | 11 | [env] 12 | PORT = '8080' 13 | 14 | [http_service] 15 | internal_port = 8080 16 | force_https = true 17 | auto_stop_machines = 'stop' 18 | auto_start_machines = true 19 | min_machines_running = 1 20 | processes = ['app'] 21 | 22 | [[vm]] 23 | memory = '2gb' 24 | cpu_kind = 'shared' 25 | cpus = 2 26 | -------------------------------------------------------------------------------- /fly-prod-kunnskap.toml: -------------------------------------------------------------------------------- 1 | # fly.toml app configuration file generated for kunnskapsassistent on 2024-09-28T12:45:18+02:00 2 | # 3 | # See https://fly.io/docs/reference/configuration/ for information about how to use this file. 4 | # 5 | 6 | app = 'kunnskapsassistent' 7 | primary_region = 'arn' 8 | 9 | [build] 10 | 11 | [env] 12 | PORT = '8080' 13 | 14 | [http_service] 15 | internal_port = 8080 16 | force_https = true 17 | auto_stop_machines = 'stop' 18 | auto_start_machines = true 19 | min_machines_running = 1 20 | processes = ['app'] 21 | 22 | [[vm]] 23 | memory = '4gb' 24 | cpu_kind = 'performance' 25 | cpus = 2 26 | -------------------------------------------------------------------------------- /fly-test-kunnskap.toml: -------------------------------------------------------------------------------- 1 | # fly.toml app configuration file generated for staging-kunnskapsassistent on 2024-09-28T10:44:22+02:00 2 | # 3 | # See https://fly.io/docs/reference/configuration/ for information about how to use this file. 4 | # 5 | 6 | app = 'test-kunnskapsassistent' 7 | primary_region = 'arn' 8 | 9 | [build] 10 | 11 | [env] 12 | PORT = '8080' 13 | 14 | [http_service] 15 | internal_port = 8080 16 | force_https = true 17 | auto_stop_machines = 'stop' 18 | auto_start_machines = true 19 | min_machines_running = 1 20 | processes = ['app'] 21 | 22 | [[vm]] 23 | memory = '2gb' 24 | cpu_kind = 'performance' 25 | cpus = 1 26 | -------------------------------------------------------------------------------- /fly-staging-kunnskap.toml: -------------------------------------------------------------------------------- 1 | # fly.toml app configuration file generated for staging-kunnskapsassistent on 2024-09-28T10:44:22+02:00 2 | # 3 | # See https://fly.io/docs/reference/configuration/ for information about how to use this file. 4 | # 5 | 6 | app = 'staging-kunnskapsassistent' 7 | primary_region = 'arn' 8 | 9 | [build] 10 | 11 | [env] 12 | PORT = '8080' 13 | 14 | [http_service] 15 | internal_port = 8080 16 | force_https = true 17 | auto_stop_machines = 'stop' 18 | auto_start_machines = true 19 | min_machines_running = 1 20 | processes = ['app'] 21 | 22 | [[vm]] 23 | memory = '2gb' 24 | cpu_kind = 'performance' 25 | cpus = 1 26 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Please add user editor configs to system gitignore: 2 | # git config --global core.excludesfile 3 | # git config --global core.excludesfile ~/.gitignore 4 | # see also: https://gist.github.com/subfuzion/db7f57fff2fb6998a16c 5 | .clj-kondo/ 6 | .clj-kondo/.cache 7 | .cpcache 8 | .lsp 9 | .nrepl-port 10 | .shadow-cljs 11 | node_modules 12 | **/resources/public/**/js 13 | **/target 14 | **/app.jar 15 | .DS_Store 16 | 17 | 18 | **/resources/public/**/styles.css 19 | **/.calva/output-window/ 20 | **/datahike-db 21 | **/target/ 22 | **/app.jar 23 | **/electric-manifest.edn 24 | 25 | pdf_inbox/* 26 | typesense_chunks/* 27 | 28 | **/secrets.edn 29 | **/mise.*.local.toml 30 | 31 | config/ka_dev.edn 32 | -------------------------------------------------------------------------------- /resources/public/chat_app/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | digdir.cloud 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /src-dev/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | %highlight(%-5level) %logger: %msg%n 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /src/chat_app/kit.cljc: -------------------------------------------------------------------------------- 1 | (ns chat-app.kit 2 | "The `kit` namespace offers a collection of essential, task-oriented utilities. 3 | These modular tools are self-contained and designed for broad reuse, enabling clear, reliable operations 4 | that focus on simplicity and functionality." 5 | (:require [clojure.string :as str] 6 | #?(:clj [nextjournal.markdown :as md]) 7 | #?(:clj [hiccup2.core :as h]) 8 | #?(:clj [nextjournal.markdown.transform :as md.transform]))) 9 | 10 | (defn lowercase-includes? [s1 s2] 11 | (and (string? s1) (string? s2) 12 | (str/includes? (str/lower-case s1) (str/lower-case s2)))) 13 | 14 | #?(:clj (defn parse-text [s] 15 | (->> (md/parse s) 16 | md.transform/->hiccup 17 | h/html 18 | str))) -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "electric-chat", 3 | "packageManager": "yarn@4.6.0", 4 | "scripts": { 5 | "build:tailwind:dev": "npx tailwindcss -i ./resources/input.css -o ./resources/public/chat_app/styles.css --watch", 6 | "build:tailwind": "npx tailwindcss -i ./resources/input.css -o ./resources/public/chat_app/styles.css --minify" 7 | }, 8 | "devDependencies": { 9 | "shadow-cljs": "2.28.21", 10 | "tailwindcss": "^3.4.17" 11 | }, 12 | "dependencies": { 13 | "@cljs-oss/module-deps": "^1.1.1", 14 | "@js-joda/core": "^5.6.3", 15 | "@js-joda/locale_en-us": "^4.14.0", 16 | "@js-joda/timezone": "^2.21.1", 17 | "katex": "^0.16.0", 18 | "markdown-it": "^12.2.0", 19 | "markdown-it-block-image": "^0.0.3", 20 | "markdown-it-footnote": "^3.0.3", 21 | "markdown-it-texmath": "^0.9.1", 22 | "markdown-it-toc-done-right": "^4.2.0", 23 | "nanoid": "^5.0.9", 24 | "punycode": "2.3.1" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # install js deps 2 | FROM node:20-slim AS node-deps 3 | WORKDIR /app 4 | COPY package.json yarn.lock .yarnrc.yml .yarn ./ 5 | RUN corepack enable && corepack prepare yarn@4.6.0 --activate && yarn install 6 | 7 | # build stage 8 | ENV JAVA_TOOL_OPTIONS="-Xmx3g" 9 | FROM clojure:temurin-21-tools-deps-bullseye-slim AS build 10 | WORKDIR /app 11 | COPY --from=node-deps /app/node_modules /app/node_modules 12 | COPY deps.edn shadow-cljs.edn package.json secrets.edn ./ 13 | 14 | COPY resources/ resources/ 15 | COPY src-build/ src-build/ 16 | COPY src-prod/ src-prod/ 17 | COPY config/ config/ 18 | COPY src/ src/ 19 | RUN clojure -A:build:prod -M -e ::ok 20 | RUN clj -X:build:prod uberjar :build/jar-name app.jar 21 | 22 | # runtime stage 23 | FROM eclipse-temurin:21.0.6_7-jre-jammy 24 | WORKDIR /app 25 | COPY --from=build /app /app 26 | #COPY --from=build /app /app/resources/public/chat_app/js/manifest.edn 27 | 28 | CMD ["java", "-cp", "app.jar", "clojure.main", "-m", "prod"] -------------------------------------------------------------------------------- /resources/public/chat_app/digdir_icon.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /shadow-cljs.edn: -------------------------------------------------------------------------------- 1 | {:builds 2 | {:dev {:target :browser 3 | :devtools {:loader-mode :default, :watch-dir "resources/public/chat_app"} 4 | :output-dir "resources/public/chat_app/js" 5 | :asset-path "/js" 6 | :module-loader true 7 | :modules {:main {:entries [dev] 8 | :init-fn dev/start! 9 | ;; :depends-on #{:webauthn} 10 | } 11 | ;; :webauthn {:entries [chat-app.webauthn]} 12 | } 13 | :build-hooks [(hyperfiddle.electric.shadow-cljs.hooks/reload-clj)]} 14 | :prod {:target :browser 15 | :output-dir "resources/public/chat_app/js" 16 | :asset-path "/js" 17 | :module-loader true 18 | :modules {:main {:entries [prod] 19 | :init-fn prod/start! 20 | ;; :depends-on #{:webauthn} 21 | } 22 | ;; :webauthn {:entries [chat-app.webauthn]} 23 | } 24 | :module-hash-names true}}} 25 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. -------------------------------------------------------------------------------- /mise.toml: -------------------------------------------------------------------------------- 1 | [env] 2 | 3 | # Assistent entity config file path. 4 | # Example: config/ka_dev.edn 5 | ENTITY_CONFIG_FILE="" 6 | 7 | # Postgres secrets 8 | ADH_POSTGRES_URL="" 9 | ADH_POSTGRES_HOST="" 10 | ADH_POSTGRES_TABLE="" 11 | ADH_POSTGRES_USER="" 12 | ADH_POSTGRES_PWD="" 13 | 14 | # Typesense secrets 15 | TYPESENSE_API_HOST="" 16 | TYPESENSE_API_KEY="" 17 | TYPESENSE_API_KEY_ADMIN="" 18 | 19 | # Azure OpenAI settings 20 | USE_AZURE_OPENAI_API=true 21 | AZURE_OPENAI_API_VERSION="2024-08-01-preview" 22 | AZURE_OPENAI_DEPLOYMENT_NAME="gpt-4o-2024-11-20" 23 | AZURE_OPENAI_MODEL_NAME="gpt-4o-2024-11-20" 24 | 25 | # Azure OpenAI secrets 26 | AZURE_OPENAI_API_ENDPOINT="" 27 | AZURE_OPENAI_API_KEY="" 28 | 29 | # OpenAI secrets 30 | OPENAI_API_URL="" 31 | OPENAI_API_MODEL_NAME="" 32 | OPENAI_API_API_KEY="" 33 | 34 | # Context window settings 35 | MAX_CONTEXT_DOC_COUNT=10 36 | MAX_CONTEXT_LENGTH=90000 37 | MAX_SOURCE_LENGTH=40000 38 | 39 | # ColBERT reranker 40 | COLBERT_API_URL="" 41 | 42 | # Postmark email service 43 | POSTMARK_API_KEY="" 44 | 45 | # Auth secrets 46 | ADMIN_USER_EMAILS="" 47 | ALLOWED_DOMAINS="" 48 | -------------------------------------------------------------------------------- /config/ai_guide_dev.edn: -------------------------------------------------------------------------------- 1 | {:db-env :remote 2 | :db {:mem {:store {:backend :mem :id "schemaless"} 3 | :schema-flexibility :read} 4 | :local {:store {:backend :file 5 | :path "./datahike-db" 6 | :id "schemaless"} 7 | :schema-flexibility :read} 8 | :remote {:store {:backend :jdbc 9 | :dbtype "postgresql" 10 | :host "aws-0-eu-central-1.pooler.supabase.com" 11 | :port 6543 12 | :dbname "postgres" 13 | :table #env ADH_POSTGRES_TABLE 14 | :user #env ADH_POSTGRES_USER 15 | :password #env ADH_POSTGRES_PWD 16 | :jdbcUrl "jdbc:postgresql://aws-0-eu-central-1.pooler.supabase.com:6543/postgres?pgbouncer=true&sslmode=require&prepareThreshold=0"} 17 | :schema-flexibility :read} 18 | 19 | :distributed {}} 20 | :chat {:all-entities-image "bot.svg" 21 | :entities [{:id "7i8dadbe-0101-f0e1-92b8-40b10a61cdcd" 22 | :name "AI Guide - dev" 23 | :image "ai-guide-logo.jpg" 24 | :docs-collection "AI-GUIDE_docs_2024-10-28" 25 | :chunks-collection "AI-GUIDE_chunks_2024-10-28" 26 | :phrases-collection "AI-GUIDE_phrases_2024-10-28" 27 | :phrase-gen-prompt "keyword-search" 28 | :reasoning-languages ["en" "no"] 29 | :prompt ""}]}} 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kunnskapsassistent (prototype) 2 | 3 | An AI tool for knowledge workers to analyze large document repositories. 4 | 5 | RAG techniques ensure grounded responses to user queries. 6 | 7 | 8 | Tech stack: 9 | - Electric Clojure v2 10 | - Typesense v28 11 | - Datahike + Postgres 12 | - OpenAI or Azure OpenAI compatible inference API 13 | - ColBERT reranker 14 | 15 | ## Local development 16 | 17 | ## Global dependencies 18 | 19 | 20 | ### Java and Clojure 21 | 22 | MacOS, using Homebrew 23 | 24 | `brew install openjdk@21` 25 | 26 | NB: be sure to add the necessary symlink as instructed at the end of the previous command 27 | 28 | Clojure command line tools: 29 | 30 | `brew install clojure/tools/clojure` 31 | 32 | 33 | ### Yarn 4.x.x 34 | `corepack enable` 35 | `yarn set version stable` 36 | 37 | 38 | ### Install project dependencies 39 | 40 | `yarn install` to install Tailwind and other javascript dependencies 41 | 42 | 43 | `yarn build:tailwind:dev` to build the css watch and build 44 | 45 | 46 | Dev build: 47 | 48 | * Shell: `clj -A:dev -X dev/-main`, or repl: `(dev/-main)` 49 | * http://localhost:8080 50 | * Hot code reloading works: edit -> save -> see app reload in browser 51 | 52 | Production build: 53 | 54 | ```shell 55 | clj -X:build:prod build-client 56 | clj -M:prod -m prod 57 | ``` 58 | 59 | ## Docker 60 | 61 | see `Dockerfile` 62 | 63 | 64 | ### Runtime env variables: 65 | 66 | For local development, you can specify values for Postgres, Typesense, Azure OpenAI and ColBERT reranker secrets in the `mise.development.local.toml` file (gitignored). 67 | 68 | In production, specify the secrets in the environment variables of the server. -------------------------------------------------------------------------------- /src-dev/dev.cljc: -------------------------------------------------------------------------------- 1 | (ns dev 2 | (:require 3 | chat-app.main 4 | [hyperfiddle.electric :as e] 5 | #?(:clj [chat-app.server-jetty :as jetty]) 6 | #?(:clj [shadow.cljs.devtools.api :as shadow]) 7 | #?(:clj [shadow.cljs.devtools.server :as shadow-server]) 8 | #?(:clj [clojure.tools.logging :as log]))) 9 | 10 | (comment (-main)) ; repl entrypoint 11 | 12 | #?(:clj ;; Server Entrypoint 13 | (do 14 | (def config 15 | {:host "0.0.0.0" 16 | :port 8080 17 | :resources-path "public/chat_app" 18 | :manifest-path ; contains Electric compiled program's version so client and server stays in sync 19 | "public/chat_app/js/manifest.edn"}) 20 | 21 | (defn -main [& args] 22 | (log/info "Starting Electric compiler and server...") 23 | 24 | (shadow-server/start!) 25 | (shadow/watch :dev) 26 | (comment (shadow-server/stop!)) 27 | 28 | (def server (jetty/start-server! 29 | (fn [ring-request] 30 | (e/boot-server {} chat-app.main/Main ring-request)) 31 | config)) 32 | 33 | (comment (.stop server)) 34 | ))) 35 | 36 | #?(:cljs ;; Client Entrypoint 37 | (do 38 | (def electric-entrypoint (e/boot-client {} chat-app.main/Main nil)) 39 | 40 | (defonce reactor nil) 41 | 42 | (defn ^:dev/after-load ^:export start! [] 43 | (set! reactor (electric-entrypoint 44 | #(js/console.log "Reactor success:" %) 45 | #(js/console.error "Reactor failure:" %)))) 46 | 47 | (defn ^:dev/before-load stop! [] 48 | (when reactor (reactor)) ; stop the reactor 49 | (set! reactor nil)))) 50 | -------------------------------------------------------------------------------- /src/chat_app/rhizome.cljc: -------------------------------------------------------------------------------- 1 | (ns chat-app.rhizome 2 | "The `rhizome` namespace contains context-aware functions that dynamically interact with the environment. 3 | Like an interconnected root system, these functions adapt and respond to platform, user, and system contexts, 4 | providing a living interface between the system and its surroundings." 5 | #?(:cljs (:require [goog.userAgent :as ua] 6 | [goog.labs.userAgent.platform :as platform]))) 7 | 8 | #?(:cljs (defn mobile-device? [] 9 | (or (or ua/IPHONE 10 | ua/PLATFORM_KNOWN_ ua/ASSUME_IPHONE 11 | (platform/isIphone)) 12 | (or ua/ANDROID 13 | ua/PLATFORM_KNOWN_ ua/ASSUME_ANDROID 14 | (platform/isAndroid))))) 15 | 16 | #?(:cljs (defn copy-to-clipboard [text] 17 | (if (and (exists? js/navigator.clipboard) 18 | (exists? js/navigator.clipboard.writeText)) 19 | (let [promise (.writeText (.-clipboard js/navigator) text)] 20 | (if (exists? (.-then promise)) 21 | (.then promise 22 | (fn [] (js/console.log "Text copied to clipboard!")) 23 | (fn [err] (js/console.error "Failed to copy text to clipboard:" err))) 24 | (js/console.error "writeText did not return a Promise"))) 25 | (js/console.error "Clipboard API not supported in this browser")))) 26 | 27 | #?(:cljs (defn speak-text [text] 28 | (if (exists? js/window.speechSynthesis) 29 | (let [utterance (js/SpeechSynthesisUtterance. text)] 30 | (.speak js/window.speechSynthesis utterance)) 31 | (js/console.error "Speech Synthesis API is not supported in this browser")))) 32 | 33 | #?(:cljs (defn pretty-print [data] 34 | (with-out-str (cljs.pprint/pprint data)))) 35 | -------------------------------------------------------------------------------- /src-prod/prod.cljc: -------------------------------------------------------------------------------- 1 | (ns prod 2 | (:require 3 | #?(:clj [clojure.edn :as edn]) 4 | #?(:clj [clojure.java.io :as io]) 5 | #?(:clj [clojure.tools.logging :as log]) 6 | [contrib.assert :refer [check]] 7 | chat-app.main 8 | #?(:clj [chat-app.server-jetty :as jetty]) 9 | [hyperfiddle.electric :as e]) 10 | #?(:cljs (:require-macros [prod :refer [compile-time-resource]]))) 11 | 12 | (defmacro compile-time-resource [filename] (some-> filename io/resource slurp edn/read-string)) 13 | 14 | (def config 15 | (merge 16 | ;; Client program's version and server program's versions must match in prod (dev is not concerned) 17 | ;; `src-build/build.clj` will compute the common version and store it in `resources/electric-manifest.edn` 18 | ;; On prod boot, `electric-manifest.edn`'s content is injected here. 19 | ;; Server is therefore aware of the program version. 20 | ;; The client's version is injected in the compiled .js file. 21 | (doto (compile-time-resource "electric-manifest.edn") prn) 22 | {:host "0.0.0.0", :port 8080, 23 | :resources-path "public/chat_app" 24 | ;; shadow build manifest path, to get the fingerprinted main.sha1.js file to ensure cache invalidation 25 | :manifest-path "public/chat_app/js/manifest.edn"})) 26 | 27 | ;;; Prod server entrypoint 28 | 29 | #?(:clj 30 | (defn -main [& {:strs [] :as args}] ; clojure.main entrypoint, args are strings 31 | (log/info (pr-str config)) 32 | (check string? (::e/user-version config)) 33 | (jetty/start-server! 34 | (fn [ring-req] (e/boot-server {} chat-app.main/Main ring-req)) 35 | config))) 36 | 37 | ;;; Prod client entrypoint 38 | 39 | #?(:cljs 40 | (do 41 | (def electric-entrypoint (e/boot-client {} chat-app.main/Main nil)) 42 | (defn ^:export start! [] 43 | (electric-entrypoint 44 | #(js/console.log "Reactor success:" %) 45 | #(js/console.error "Reactor failure:" %))))) 46 | -------------------------------------------------------------------------------- /resources/input.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; 4 | 5 | html, body { 6 | height: 100%; 7 | } 8 | 9 | ::-webkit-scrollbar-track { 10 | background-color: transparent; 11 | } 12 | 13 | ::-webkit-scrollbar-thumb { 14 | background-color: #ccc; 15 | border-radius: 10px; 16 | } 17 | 18 | ::-webkit-scrollbar-thumb:hover { 19 | background-color: #aaa; 20 | } 21 | 22 | ::-webkit-scrollbar-track:hover { 23 | background-color: #f2f2f2; 24 | } 25 | 26 | ::-webkit-scrollbar-corner { 27 | background-color: transparent; 28 | } 29 | 30 | ::-webkit-scrollbar { 31 | width: 6px; 32 | height: 6px; 33 | } 34 | /* 35 | html { 36 | background: #202123; 37 | } */ 38 | 39 | @media (max-width: 720px) { 40 | pre { 41 | /* width: calc(100vw - 110px); */ 42 | } 43 | } 44 | 45 | pre:has(div.codeblock) { 46 | padding: 0; 47 | } 48 | pre:has(code) { 49 | padding-top: 0.6rem; 50 | /* padding-bottom: 0.0rem; */ 51 | background-color: lightgray; 52 | font-size: 0.8em; 53 | line-height: 130%; 54 | overflow-x: scroll; 55 | } 56 | 57 | /* markdown styles */ 58 | ol { 59 | padding-top: 8px; 60 | padding-bottom: 10px; 61 | } 62 | 63 | ul { 64 | padding-left: 4px; 65 | padding-top: 0.6rem; 66 | padding-bottom: 0rem; 67 | } 68 | 69 | div ul { 70 | padding-top: 0.6rem; 71 | } 72 | 73 | 74 | ol ul { 75 | padding-top: 0.2rem; 76 | } 77 | 78 | li { 79 | padding-top: 0.3rem; 80 | padding-bottom: 0px; 81 | /* list-style-type: disc; */ 82 | 83 | } 84 | 85 | ol li { 86 | padding-top: 0.7rem; 87 | } 88 | 89 | ol ul li { 90 | list-style-type: disc; 91 | position: relative; 92 | margin-left: 1rem; 93 | margin-top: 0.1rem; 94 | } 95 | 96 | li ul li { 97 | padding-top: 0.1rem; 98 | } 99 | /* end markdown styles */ 100 | 101 | .anchored-scroller * { 102 | overflow-anchor: none; 103 | } 104 | 105 | div.scroller-anchor { 106 | overflow-anchor: auto; 107 | min-height: 1px; 108 | } 109 | 110 | a { 111 | color: blue; 112 | } 113 | 114 | .animate-spin { 115 | animation: spin 3s linear; 116 | opacity: 1; 117 | transition: opacity 0.3s ease-out; 118 | } 119 | 120 | @keyframes spin { 121 | from { 122 | transform: rotate(0deg); 123 | } 124 | to { 125 | transform: rotate(360deg); 126 | } 127 | } 128 | 129 | .animate-spin:not(:active) { 130 | opacity: 0; 131 | } -------------------------------------------------------------------------------- /src/chat_app/main.cljc: -------------------------------------------------------------------------------- 1 | (ns chat-app.main 2 | (:require [chat-app.debug :as debug] 3 | #?(:clj [services.system :as system]) 4 | #?(:clj [models.db]) 5 | #?(:clj [chat-app.rag :as rag]) 6 | #?(:clj [chat-app.auth :as auth]) 7 | [chat-app.webauthn :as webauthn] 8 | [chat-app.ui :as ui] 9 | [chat-app.chat :as chat] 10 | [chat-app.conversations :as conversations] 11 | [chat-app.config-ui :as config-ui] 12 | [chat-app.auth-ui :as auth-ui] 13 | 14 | [hyperfiddle.electric :as e] 15 | [hyperfiddle.electric-dom2 :as dom] 16 | [hyperfiddle.rcf :refer [tests tap %]] 17 | 18 | ;; rag tests 19 | #?(:clj [chat-app.rag-test]))) 20 | 21 | (defn T 22 | "For debugging 23 | Input → ___ → Output 24 | | 25 | | 26 | ↓ 27 | Console" 28 | ([x] 29 | (prn x) 30 | x) 31 | ([tag x] 32 | (prn tag x) 33 | x)) 34 | 35 | (hyperfiddle.rcf/enable!) 36 | 37 | (defn toggle [s k] 38 | (if (s k) 39 | (disj s k) 40 | (conj s k))) 41 | 42 | (e/defn MainView [debug-props] 43 | (e/client 44 | (dom/div (dom/props {:class "flex flex-1 h-full w-full"}) 45 | (dom/div (dom/props {:class "relative flex-1 overflow-hidden pb-[110px]"}) 46 | (case (e/watch ui/!view-main) 47 | :home (ui/Home.) 48 | :conversation (chat/Conversation.) 49 | :dashboard (auth-ui/AuthAdminDashboard.) 50 | :edit-prompt (config-ui/PromptEditor.)) 51 | (when (e/watch debug/!debug?) 52 | (debug/DBInspector. debug-props)))))) 53 | 54 | 55 | (e/defn Main [ring-request] 56 | (e/server 57 | (binding [e/http-request ring-request] 58 | (e/client 59 | (binding [dom/node js/document.body] 60 | (ui/Topbar. (e/watch chat/!conversation-entity)) 61 | (dom/main (dom/props {:class "flex h-full w-screen flex-col text-sm absolute top-0 pt-12"}) 62 | (dom/div (dom/props {:class "flex h-full w-full pt-[48px] sm:pt-0 items-start"}) 63 | (conversations/LeftSidebar.) 64 | (MainView. {:!active-conversation chat/!active-conversation 65 | :!conversation-entity chat/!conversation-entity}) 66 | (debug/DebugController.)))))))) 67 | -------------------------------------------------------------------------------- /src/services/openai.cljc: -------------------------------------------------------------------------------- 1 | (ns services.openai 2 | (:require #?(:clj [models.db :refer [conn 3 | transact-assistant-msg]]) 4 | #?(:clj [wkok.openai-clojure.api :as api]))) 5 | 6 | #?(:clj (defn process-chunk [!stream-msgs] 7 | (fn [convo-id data] 8 | (let [delta (get-in data [:choices 0 :delta]) 9 | content (:content delta)] 10 | (if content 11 | (swap! !stream-msgs update-in [convo-id :content] (fn [old-content] (str old-content content))) 12 | (do 13 | (swap! !stream-msgs assoc-in [convo-id :streaming] false) 14 | (let [resp (:content (get @!stream-msgs convo-id))] 15 | (transact-assistant-msg conn convo-id resp)) 16 | (swap! !stream-msgs assoc-in [convo-id :content] nil))))))) 17 | 18 | #?(:clj (defn stream-chat-completion [!stream-msgs convo-id msg-list] 19 | (swap! !stream-msgs assoc-in [convo-id :streaming] true) 20 | (let [process-chunk-fn (process-chunk !stream-msgs)] 21 | (try (api/create-chat-completion 22 | {:model "gpt-4o" 23 | :messages msg-list 24 | :stream true 25 | :on-next #(process-chunk-fn convo-id %)}) 26 | (catch Exception e 27 | (println "This is the exception: " e)))))) 28 | 29 | #?(:clj (defn get-chat-completion [!wait? convo-id msg-list] 30 | (let [_ (reset! !wait? true) 31 | _ (println "reset wait to true") 32 | _ (println "the msg list: " msg-list) 33 | raw-resp (api/create-chat-completion {:model "gpt-4o" 34 | :messages msg-list}) 35 | resp (get-in raw-resp [:choices 0 :message :content])] 36 | (transact-assistant-msg conn convo-id resp) 37 | (reset! !wait? false) 38 | (println "reset wait to false")))) 39 | 40 | #?(:clj (defn use-azure-openai [] (not= "false" (System/getenv "USE_AZURE_OPENAI_API")))) 41 | 42 | #?(:clj 43 | (defn create-chat-completion [messages] 44 | (if (use-azure-openai) 45 | (api/create-chat-completion 46 | {:model (System/getenv "AZURE_OPENAI_DEPLOYMENT_NAME") 47 | :messages messages 48 | :temperature 0.1 49 | :max_tokens nil} 50 | {:api-key (System/getenv "AZURE_OPENAI_API_KEY") 51 | :api-endpoint (System/getenv "AZURE_OPENAI_ENDPOINT") 52 | :impl :azure}) 53 | (api/create-chat-completion 54 | {:model (System/getenv "OPENAI_API_MODEL_NAME") 55 | :messages messages 56 | :temperature 0.1 57 | :stream false 58 | :max_tokens nil})))) -------------------------------------------------------------------------------- /src/services/system.cljc: -------------------------------------------------------------------------------- 1 | (ns services.system 2 | (:require #?(:clj [models.db :as db :refer [delayed-connection]]) 3 | #?(:clj [chat-app.rag :as rag]))) 4 | 5 | #?(:clj 6 | (defn answer-user-query-with-rag [{:keys [conversation-entity convo-id user-query] :as params}] 7 | #_(swap! !stream-msgs assoc-in [convo-id :streaming] true) 8 | (let [conn @delayed-connection 9 | _ (reset! rag/!response-states ["Vurderer spørsmålet ditt"]) 10 | 11 | {:keys [id 12 | docs-collection chunks-collection 13 | phrases-collection phrase-gen-prompt 14 | promptRagQueryRelax promptRagGenerate]} conversation-entity 15 | 16 | rag-response (rag/rag-pipeline 17 | {;; :on-next #(process-chunk convo-id %) 18 | :conversation-id convo-id 19 | :entity-id id 20 | :translated_user_query user-query 21 | :original_user_query user-query 22 | :user_query_language_name "Norwegian" 23 | :promptRagQueryRelax promptRagQueryRelax, 24 | :promptRagGenerate promptRagGenerate, 25 | :maxSourceDocCount 200 26 | :maxSourceLength 10000 27 | :maxContextLength 40000 28 | :docsCollectionName docs-collection 29 | :chunksCollectionName chunks-collection 30 | :phrasesCollectionName phrases-collection 31 | :phrase-gen-prompt phrase-gen-prompt 32 | :stream_callback_msg1 nil 33 | :stream_callback_msg2 nil 34 | :streamCallbackFreqSec 2.0 35 | :maxResponseTokenCount nil} 36 | conn) 37 | _ (reset! rag/!response-states nil)] 38 | rag-response))) 39 | 40 | 41 | ;; Add a watch function to process jobs continuously 42 | #?(:clj 43 | (defn start-job-processor! 44 | "Start a background process to handle RAG jobs" 45 | [] 46 | (add-watch rag/!rag-jobs :job-processor 47 | (fn [_ _ _ jobs] 48 | (when (seq jobs) 49 | (when-let [job (rag/dequeue-rag-job)] 50 | (future 51 | (try 52 | (println "Processing job:" job) 53 | (case (:type job) 54 | :followup (answer-user-query-with-rag (:msg-data job)) 55 | :new (answer-user-query-with-rag (:msg-data job))) 56 | (catch Exception e 57 | (println "Error processing job:" e)) 58 | (finally 59 | (future 60 | (Thread/sleep 7000) 61 | (reset! rag/!response-states nil))))))))))) 62 | 63 | ;; Initialize the job processor when the namespace is loaded 64 | #?(:clj (start-job-processor!)) 65 | -------------------------------------------------------------------------------- /src-build/build.clj: -------------------------------------------------------------------------------- 1 | (ns build 2 | (:require 3 | [clojure.tools.build.api :as b] 4 | [clojure.tools.logging :as log] 5 | [shadow.cljs.devtools.api :as shadow-api] 6 | [shadow.cljs.devtools.server :as shadow-server])) 7 | 8 | (def electric-user-version "7d8ad8a-dirty" 9 | #_(b/git-process {:git-args "describe --tags --long --always --dirty"})) 10 | 11 | (defn build-client ; invoke with `clj -X ...` 12 | "build Electric app client, invoke with -X 13 | e.g. `clojure -X:build:prod build-client :debug false :verbose false :optimize true` 14 | Note: Electric shadow compilation requires application classpath to be available, so do not use `clj -T`" 15 | [argmap] 16 | (let [{:keys [optimize debug verbose] 17 | :or {optimize true, debug false, verbose false} 18 | :as config} 19 | (assoc argmap :hyperfiddle.electric/user-version electric-user-version)] 20 | (b/delete {:path "resources/public/chat_app/js"}) 21 | (b/delete {:path "resources/electric-manifest.edn"}) 22 | 23 | ; bake user-version into artifact, cljs and clj 24 | (b/write-file {:path "resources/electric-manifest.edn" :content config}) 25 | 26 | ; "java.lang.NoClassDefFoundError: com/google/common/collect/Streams" is fixed by 27 | ; adding com.google.guava/guava {:mvn/version "31.1-jre"} to deps, 28 | ; see https://hf-inc.slack.com/archives/C04TBSDFAM6/p1692636958361199 29 | (shadow-server/start!) 30 | (as-> 31 | (shadow-api/release :prod 32 | {:debug debug, 33 | :verbose verbose, 34 | :config-merge 35 | [{:compiler-options {:optimizations (if optimize :advanced :simple)} 36 | :closure-defines {'hyperfiddle.electric-client/ELECTRIC_USER_VERSION electric-user-version}}]}) 37 | shadow-status (assert (= shadow-status :done) "shadow-api/release error")) ; fail build on error 38 | (shadow-server/stop!) 39 | (log/info "Client build successful. Version:" electric-user-version))) 40 | 41 | (def class-dir "target/classes") 42 | 43 | (defn uberjar 44 | [{:keys [optimize debug verbose ::jar-name, ::skip-client] 45 | :or {optimize true, debug false, verbose false, skip-client false} 46 | :as args}] 47 | ; careful, shell quote escaping combines poorly with clj -X arg parsing, strings read as symbols 48 | (log/info `uberjar (pr-str args)) 49 | (b/delete {:path "target"}) 50 | 51 | (when-not skip-client 52 | (build-client {:optimize optimize, :debug debug, :verbose verbose})) 53 | 54 | (b/copy-dir {:target-dir class-dir :src-dirs ["src" "src-prod" "resources"]}) 55 | (let [jar-name (or (some-> jar-name str) ; override for Dockerfile builds to avoid needing to reconstruct the name 56 | (format "target/electricfiddle-%s.jar" electric-user-version)) 57 | aliases [:prod]] 58 | (log/info `uberjar "included aliases:" aliases) 59 | (b/uber {:class-dir class-dir 60 | :uber-file jar-name 61 | :basis (b/create-basis {:project "deps.edn" :aliases aliases})}) 62 | (log/info jar-name))) 63 | 64 | ;; clj -X:build:prod build-client 65 | ;; clj -X:build:prod uberjar :build/jar-name "app.jar" 66 | ;; java -cp app.jar clojure.main -m prod 67 | -------------------------------------------------------------------------------- /src/chat_app/config_ui.cljc: -------------------------------------------------------------------------------- 1 | (ns chat-app.config-ui 2 | (:require [hyperfiddle.electric :as e] 3 | [hyperfiddle.electric-dom2 :as dom] 4 | #?(:clj [models.db :refer [update-config] :as db]))) 5 | 6 | ;; Entity state 7 | (e/def entities-cfg 8 | (e/server (:chat (e/watch db/!config)))) 9 | 10 | (e/defn PromptEditor [] 11 | (e/client 12 | (let [cfg entities-cfg 13 | entity (when cfg (-> cfg :entities first)) 14 | {:keys [id promptRagGenerate promptRagQueryRelax]} (or entity {}) 15 | !promptRagGenerate (atom promptRagGenerate) 16 | !promptRagQueryRelax (atom promptRagQueryRelax) 17 | promptGenerate (e/watch !promptRagGenerate) 18 | promptQueryRelax (e/watch !promptRagQueryRelax)] 19 | (if-not entity 20 | (dom/div (dom/text "Loading...")) 21 | (dom/div 22 | (dom/props {:class "m-2 h-full overflow-y-auto"}) 23 | (dom/h3 (dom/text "Query relax prompt:")) 24 | (dom/div 25 | (dom/textarea 26 | (dom/props {:value (or promptQueryRelax "") 27 | :style {:width "94%" 28 | :height "200px" 29 | :font-family "monospace" 30 | :margin "10px" 31 | :padding "10px" 32 | :border "1px solid #ccc" 33 | :border-radius "4px"} 34 | :placeholder "Loading config file..."}) 35 | (dom/on "change" (e/fn [e] 36 | (when-some [v (not-empty (.. e -target -value))] 37 | (reset! !promptRagQueryRelax v))))) 38 | 39 | (dom/h3 (dom/text "Generate prompt:")) 40 | (dom/textarea 41 | (dom/props {:value (or promptGenerate "") 42 | :style {:width "94%" 43 | :height "200px" 44 | :font-family "monospace" 45 | :margin "10px" 46 | :padding "10px" 47 | :border "1px solid #ccc" 48 | :border-radius "4px"} 49 | :placeholder "Loading config file..."}) 50 | (dom/on "change" (e/fn [e] 51 | (when-some [v (not-empty (.. e -target -value))] 52 | (reset! !promptRagGenerate v))))) 53 | (dom/div (dom/props {:class (str "bottom-3" " absolute right-0 w-full border-transparent bg-gradient-to-b from-transparent via-white to-white pt-6 md:pt-2")}) 54 | (dom/button 55 | (dom/on "click" 56 | (e/fn [_] 57 | (println "Saving prompts") 58 | (e/server 59 | (e/offload 60 | #(update-config 61 | {:id id 62 | :promptRagGenerate promptGenerate 63 | :promptRagQueryRelax promptQueryRelax})) 64 | nil))) 65 | (dom/props {:class "px-4 py-2 bg-blue-500 text-white rounded-md hover:bg-blue-600 focus:outline-none"}) 66 | (dom/text "Save changes"))))))))) 67 | -------------------------------------------------------------------------------- /config/ka_dev.edn: -------------------------------------------------------------------------------- 1 | {:db-env :remote, 2 | :db {:mem {:store {:backend :mem :id "schemaless"} 3 | :schema-flexibility :read} 4 | :local {:store {:backend :file 5 | :path "./datahike-db" 6 | :id "schemaless"} 7 | :schema-flexibility :read} 8 | :remote {:store {:backend :jdbc 9 | :dbtype "postgresql" 10 | :host #env ADH_POSTGRES_HOST 11 | :port 6543 12 | :dbname "postgres" 13 | :table #env ADH_POSTGRES_TABLE 14 | :user #env ADH_POSTGRES_USER 15 | :password #env ADH_POSTGRES_PWD 16 | :jdbcUrl #env ADH_POSTGRES_URL} 17 | :schema-flexibility :read 18 | :keep-history? false} 19 | :distributed {}}, 20 | :chat 21 | {:all-entities-image "bot.svg", 22 | :entities 23 | ({:docs-collection "TEST_kudos_docs", 24 | :name "Kunnskapsassistent - DEV", 25 | :phrases-collection "TEST_kudos_phrases", 26 | :promptRagQueryRelax 27 | "You have access to a search API that returns relevant documentation. 28 | Your task is to generate an array of up to 7 search queries that are relevant to the current state of the conversation provided below. 29 | 30 | Use a variation of related keywords and synonyms for the queries, trying to be as general as possible. 31 | Include as many queries as you can think of, including and excluding terms. For example, include queries like [\"keyword_1 keyword_2\", \"keyword_1\", \"keyword_2\"]. Be creative. The more queries you include, the more likely you are to find relevant results. 32 | 33 | 34 | {messages} 35 | 36 | ", 37 | :promptRagGenerate 38 | "Du kan se på meg som en kunnskapsassistent. Klar for å svare på spørsmål du har om offentlig sektor. 39 | Du har tilgang til: 40 | - Kudos databasen med mer enn 3000 kunnskaps- og styrings- dokumenter 41 | - Foreløpig har du kun årsrapporter, tildelingsbrev og evalueringer fra 2020 til og med 2023 og statusrapporter fra 2020-2024. 42 | - Vi har søkt med følgende filtere: 43 | Dokument type: [\"Årsrapport\" \"Tildelingsbrev\" \"Evaluering\" \"Statusrapport\"] 44 | Vi skal forsøke å gi deg mange relevante avsnitt fra dokumenter i databasen vår, men det kan hende at noen av de er ikke relevant for brukerens spørsmål. Hvis du ikke vet svaret, si at du ikke vet. 45 | 46 | 47 | {context} 48 | 49 | Spørsmål: {question} 50 | Gi det hjelpsomme svaret nedenfor. 51 | Hvis spørsmålet er uklart, be om presisering, før du gir endelig svar 52 | Hvis bruker spør hvilke dokumenter du har tilgang til, si at du har den nevnte dokumenttypen til statlige virksomheter fra 2020-2023 (2020-2024 hvis statusrapport). Si at du ikke kan ramse opp alle dokumentene, og spør hvilke virksomheter de vil utforske nærmere. 53 | Bruk Markdown formattering for å gjøre svaret oversiktlig og for bedre lesbarhet. 54 | Ta med kilden i parentes, kun hvis du har brukt en kilde fra de relevante kildene. 55 | Gi alltid ditt hjelpsomme svar på norsk, med mindre spørsmålet er på et annet språk. Hvis spørsmålet er på et annet språk, gi svaret på det språket. 56 | 57 | Hjelpsomt svar: 58 | " 59 | :phrase-gen-prompt "keyword-search", 60 | :prompt "", 61 | :reasoning-languages ["en" "no"], 62 | :id "71e1adbe-2116-478d-92b8-40b10a612d7b", 63 | :image "kudos-logo.png", 64 | :chunks-collection "TEST_kudos_chunks"})}} 65 | -------------------------------------------------------------------------------- /deps.edn: -------------------------------------------------------------------------------- 1 | {:deps {org.babashka/http-client {:mvn/version "0.3.11"} 2 | org.clojure/data.json {:mvn/version "2.5.1"} 3 | buddy/buddy-sign {:mvn/version "3.6.1-359"} 4 | tick/tick {:mvn/version "1.0"} 5 | io.replikativ/datahike {:mvn/version "0.6.1594"} 6 | io.replikativ/datahike-jdbc {:mvn/version "0.3.49"} 7 | org.postgresql/postgresql {:mvn/version "42.7.5"} 8 | net.clojars.wkok/openai-clojure {:mvn/version "0.18.1"} 9 | nano-id/nano-id {:mvn/version "1.0.0"} 10 | aero/aero {:mvn/version "1.1.6"} 11 | io.github.nextjournal/markdown {:mvn/version "0.6.157"} 12 | hiccup/hiccup {:mvn/version "2.0.0-RC4"} 13 | markdown-clj/markdown-clj {:mvn/version "1.12.3"} 14 | io.github.lambdaisland/deep-diff2 {:git/sha "302ded8f32c31c21bccbfa3848601c98187d7521"} 15 | clj-http/clj-http {:mvn/version "3.13.0"} 16 | io.github.runeanielsen/typesense-clj {:mvn/version "0.1.146"} 17 | dev.weavejester/medley {:mvn/version "1.8.1"} 18 | cheshire/cheshire {:mvn/version "5.13.0"} 19 | org.clojars.askonomm/ruuter {:mvn/version "1.3.4"} 20 | ;; Auth deps 21 | mvxcvi/clj-cbor {:mvn/version "1.1.1"} 22 | pandect/pandect {:mvn/version "1.0.2"} 23 | ;; Electric Dependencies 24 | ;; com.hyperfiddle/electric {:local/root "../vendors/electric"} 25 | com.hyperfiddle/electric {:git/url "https://github.com/hyperfiddle/electric" :git/sha "f620a2f89f8e706d1c3d42c29837bb2889e301ef"} 26 | com.hyperfiddle/rcf {:mvn/version "20220926-202227"} 27 | ring/ring {:mvn/version "1.11.0"} ; comes with Jetty 28 | org.clojure/clojure {:mvn/version "1.12.0"} ; later releases break Electric v2 29 | org.clojure/clojurescript {:mvn/version "1.11.132"} 30 | org.clojure/tools.logging {:mvn/version "1.2.4"} 31 | ch.qos.logback/logback-classic {:mvn/version "1.5.17"}} 32 | :paths ["src" "resources"] 33 | :aliases {:dev 34 | {:extra-paths ["src-dev"] 35 | :extra-deps {djblue/portal {:mvn/version "0.58.5"} 36 | thheller/shadow-cljs {:mvn/version "2.26.2"} 37 | io.github.clojure/tools.build {:mvn/version "0.10.7" 38 | :exclusions [com.google.guava/guava ; Guava version conflict between tools.build and clojurescript. 39 | org.slf4j/slf4j-nop]}}} ; clashes with app logger 40 | :cider-clj {:extra-deps {cider/cider-nrepl {:mvn/version "0.52.1"}} 41 | :main-opts ["-m" "nrepl.cmdline" "--middleware" "[cider.nrepl/cider-middleware]"]} 42 | :prod 43 | {:extra-paths ["src-prod"]} 44 | 45 | :build ; use `clj -X:build build-client`, NOT -T! build/app classpath contamination cannot be prevented 46 | {:extra-paths ["src-build"] 47 | :ns-default build 48 | :extra-deps {thheller/shadow-cljs {:mvn/version "2.26.2"} 49 | io.github.clojure/tools.build {:mvn/version "0.10.7" 50 | :exclusions [com.google.guava/guava ; Guava version conflict between tools.build and clojurescript. 51 | org.slf4j/slf4j-nop]}}} ; clashes with app logger 52 | }} 53 | -------------------------------------------------------------------------------- /config/ka_staging.edn: -------------------------------------------------------------------------------- 1 | {:db-env :remote 2 | :db {:mem {:store {:backend :mem :id "schemaless"} 3 | :schema-flexibility :read} 4 | :local {:store {:backend :file 5 | :path "./datahike-db" 6 | :id "schemaless"} 7 | :schema-flexibility :read} 8 | :remote {:store {:backend :jdbc 9 | :dbtype "postgresql" 10 | :host #env ADH_POSTGRES_HOST 11 | :port 6543 12 | :dbname "postgres" 13 | :table #env ADH_POSTGRES_TABLE 14 | :user #env ADH_POSTGRES_USER 15 | :password #env ADH_POSTGRES_PWD 16 | :jdbcUrl #env ADH_POSTGRES_URL} 17 | :schema-flexibility :read} 18 | 19 | :distributed {}} 20 | :chat {:all-entities-image "bot.svg" 21 | :entities [{:id "71e1adbe-2116-478d-92b8-40b10a612d7b" 22 | :name "Kunnskapsassistent - staging" 23 | :image "kudos-logo.png" 24 | :docs-collection "STAGING_kudos_docs" 25 | :chunks-collection "STAGING_kudos_chunks" 26 | :phrases-collection "STAGING_kudos_phrases" 27 | :phrase-gen-prompt "keyword-search" 28 | :promptRagQueryRelax "You have access to a search API that returns relevant documentation. 29 | Your task is to generate an array of up to 7 search queries that are relevant to the current state of the conversation provided below. 30 | Use a variation of related keywords and synonyms for the queries, trying to be as general as possible. 31 | Include as many queries as you can think of, including and excluding terms. For example, include queries like [\"keyword_1 keyword_2\", \"keyword_1\", \"keyword_2\"]. Be creative. The more queries you include, the more likely you are to find relevant results. 32 | 33 | {messages} 34 | 35 | ", 36 | :promptRagGenerate 37 | "Du kan se på meg som en kunnskapsassistent. Klar for å svare på spørsmål du har om offentlig sektor. 38 | Du har tilgang til: 39 | - Kudos databasen med mer enn 3000 kunnskaps- og styrings- dokumenter 40 | - Foreløpig har du kun årsrapporter, tildelingsbrev og evalueringer fra 2020 til og med 2023 og statusrapporter fra 2020-2024. 41 | - Vi har søkt med følgende filtere: 42 | Dokument type: [\"Årsrapport\" \"Tildelingsbrev\" \"Evaluering\" \"Statusrapport\"] 43 | Vi skal forsøke å gi deg mange relevante avsnitt fra dokumenter i databasen vår, men det kan hende at noen av de er ikke relevant for brukerens spørsmål. Hvis du ikke vet svaret, si at du ikke vet. 44 | 45 | 46 | {context} 47 | 48 | Spørsmål: {question} 49 | Gi det hjelpsomme svaret nedenfor. 50 | Hvis spørsmålet er uklart, be om presisering, før du gir endelig svar 51 | Hvis bruker spør hvilke dokumenter du har tilgang til, si at du har den nevnte dokumenttypen til statlige virksomheter fra 2020-2023 (2020-2024 hvis statusrapport). Si at du ikke kan ramse opp alle dokumentene, og spør hvilke virksomheter de vil utforske nærmere. 52 | Bruk Markdown formattering for å gjøre svaret oversiktlig og for bedre lesbarhet. 53 | Ta med kilden i parentes, kun hvis du har brukt en kilde fra de relevante kildene. 54 | Gi alltid ditt hjelpsomme svar på norsk, med mindre spørsmålet er på et annet språk. Hvis spørsmålet er på et annet språk, gi svaret på det språket. 55 | 56 | Hjelpsomt svar: 57 | " 58 | :reasoning-languages ["en" "no"] 59 | :prompt ""}]}} 60 | -------------------------------------------------------------------------------- /config/ka_test.edn: -------------------------------------------------------------------------------- 1 | {:db-env :remote 2 | :db {:mem {:store {:backend :mem :id "schemaless"} 3 | :schema-flexibility :read} 4 | :local {:store {:backend :file 5 | :path "./datahike-db" 6 | :id "schemaless"} 7 | :schema-flexibility :read} 8 | :remote {:store {:backend :jdbc 9 | :dbtype "postgresql" 10 | :host #env ADH_POSTGRES_HOST 11 | :port 6543 12 | :dbname "postgres" 13 | :table #env ADH_POSTGRES_TABLE 14 | :user #env ADH_POSTGRES_USER 15 | :password #env ADH_POSTGRES_PWD 16 | :jdbcUrl #env ADH_POSTGRES_URL} 17 | :schema-flexibility :read} 18 | 19 | :distributed {}} 20 | :chat {:all-entities-image "bot.svg" 21 | :entities [{:id "71e1adbe-2116-478d-92b8-40b10a612d7b" 22 | :name "Kunnskapsassistent - test" 23 | :image "kudos-logo.png" 24 | :docs-collection "TEST_kudos_docs" 25 | :chunks-collection "TEST_kudos_chunks" 26 | :phrases-collection "TEST_kudos_phrases" 27 | :phrase-gen-prompt "keyword-search" 28 | :promptRagQueryRelax 29 | "You have access to a search API that returns relevant documentation. 30 | Your task is to generate an array of up to 7 search queries that are relevant to the current state of the conversation provided below. 31 | 32 | Use a variation of related keywords and synonyms for the queries, trying to be as general as possible. 33 | Include as many queries as you can think of, including and excluding terms. For example, include queries like [\"keyword_1 keyword_2\", \"keyword_1\", \"keyword_2\"]. Be creative. The more queries you include, the more likely you are to find relevant results. 34 | 35 | {messages} 36 | 37 | " 38 | :promptRagGenerate 39 | "Du kan se på meg som en kunnskapsassistent. Klar for å svare på spørsmål du har om offentlig sektor. 40 | Du har tilgang til: 41 | - Kudos databasen med mer enn 3000 kunnskaps- og styrings- dokumenter 42 | - Foreløpig har du kun årsrapporter, tildelingsbrev og evalueringer fra 2020 til og med 2023 og statusrapporter fra 2020-2024. 43 | - Vi har søkt med følgende filtere: 44 | Dokument type: [\"Årsrapport\" \"Tildelingsbrev\" \"Evaluering\" \"Statusrapport\"] 45 | Vi skal forsøke å gi deg mange relevante avsnitt fra dokumenter i databasen vår, men det kan hende at noen av de er ikke relevant for brukerens spørsmål. Hvis du ikke vet svaret, si at du ikke vet. 46 | 47 | 48 | {context} 49 | 50 | Spørsmål: {question} 51 | Gi det hjelpsomme svaret nedenfor. 52 | Hvis spørsmålet er uklart, be om presisering, før du gir endelig svar 53 | Hvis bruker spør hvilke dokumenter du har tilgang til, si at du har den nevnte dokumenttypen til statlige virksomheter fra 2020-2023 (2020-2024 hvis statusrapport). Si at du ikke kan ramse opp alle dokumentene, og spør hvilke virksomheter de vil utforske nærmere. 54 | Bruk Markdown formattering for å gjøre svaret oversiktlig og for bedre lesbarhet. 55 | Ta med kilden i parentes, kun hvis du har brukt en kilde fra de relevante kildene. 56 | Gi alltid ditt hjelpsomme svar på norsk, med mindre spørsmålet er på et annet språk. Hvis spørsmålet er på et annet språk, gi svaret på det språket. 57 | 58 | Hjelpsomt svar: 59 | " 60 | :reasoning-languages ["en" "no"] 61 | :prompt ""}]}} 62 | 63 | -------------------------------------------------------------------------------- /config/ka_sandbox.edn: -------------------------------------------------------------------------------- 1 | {:db-env :remote 2 | :db {:mem {:store {:backend :mem :id "schemaless"} 3 | :schema-flexibility :read} 4 | :local {:store {:backend :file 5 | :path "./datahike-db" 6 | :id "schemaless"} 7 | :schema-flexibility :read} 8 | :remote {:store {:backend :jdbc 9 | :dbtype "postgresql" 10 | :host #env ADH_POSTGRES_HOST 11 | :port 6543 12 | :dbname "postgres" 13 | :table #env ADH_POSTGRES_TABLE 14 | :user #env ADH_POSTGRES_USER 15 | :password #env ADH_POSTGRES_PWD 16 | :jdbcUrl #env ADH_POSTGRES_URL} 17 | :schema-flexibility :read 18 | :keep-history? false 19 | } 20 | 21 | :distributed {}} 22 | :chat {:all-entities-image "bot.svg" 23 | :entities [{:id "71e1adbe-2116-478d-92b8-40b10a612d7b" 24 | :name "Kunnskapsassistent - SANDBOX" 25 | :image "kudos-logo.png" 26 | :docs-collection "TEST_kudos_docs" 27 | :chunks-collection "TEST_kudos_chunks" 28 | :phrases-collection "TEST_kudos_phrases" 29 | :phrase-gen-prompt "keyword-search" 30 | :promptRagQueryRelax "You have access to a search API that returns relevant documentation. 31 | Your task is to generate an array of up to 7 search queries that are relevant to the current state of the conversation provided below. 32 | Use a variation of related keywords and synonyms for the queries, trying to be as general as possible. 33 | Include as many queries as you can think of, including and excluding terms. For example, include queries like [\"keyword_1 keyword_2\", \"keyword_1\", \"keyword_2\"]. Be creative. The more queries you include, the more likely you are to find relevant results. 34 | 35 | {messages} 36 | 37 | ", 38 | :promptRagGenerate 39 | "Du kan se på meg som en kunnskapsassistent. Klar for å svare på spørsmål du har om offentlig sektor. 40 | Du har tilgang til: 41 | - Kudos databasen med mer enn 3000 kunnskaps- og styrings- dokumenter 42 | - Foreløpig har du kun årsrapporter, tildelingsbrev og evalueringer fra 2020 til og med 2023 og statusrapporter fra 2020-2024. 43 | - Vi har søkt med følgende filtere: 44 | Dokument type: [\"Årsrapport\" \"Tildelingsbrev\" \"Evaluering\" \"Statusrapport\"] 45 | Vi skal forsøke å gi deg mange relevante avsnitt fra dokumenter i databasen vår, men det kan hende at noen av de er ikke relevant for brukerens spørsmål. Hvis du ikke vet svaret, si at du ikke vet. 46 | 47 | 48 | {context} 49 | 50 | Spørsmål: {question} 51 | Gi det hjelpsomme svaret nedenfor. 52 | Hvis spørsmålet er uklart, be om presisering, før du gir endelig svar 53 | Hvis bruker spør hvilke dokumenter du har tilgang til, si at du har den nevnte dokumenttypen til statlige virksomheter fra 2020-2023 (2020-2024 hvis statusrapport). Si at du ikke kan ramse opp alle dokumentene, og spør hvilke virksomheter de vil utforske nærmere. 54 | Bruk Markdown formattering for å gjøre svaret oversiktlig og for bedre lesbarhet. 55 | Ta med kilden i parentes, kun hvis du har brukt en kilde fra de relevante kildene. 56 | Gi alltid ditt hjelpsomme svar på norsk, med mindre spørsmålet er på et annet språk. Hvis spørsmålet er på et annet språk, gi svaret på det språket. 57 | 58 | Hjelpsomt svar: 59 | " 60 | :reasoning-languages ["en" "no"] 61 | :prompt ""}]}} 62 | 63 | -------------------------------------------------------------------------------- /config/ka_prod.edn: -------------------------------------------------------------------------------- 1 | {:db-env :remote 2 | :db {:mem {:store {:backend :mem :id "schemaless"} 3 | :schema-flexibility :read} 4 | :local {:store {:backend :file 5 | :path "./datahike-db" 6 | :id "schemaless"} 7 | :schema-flexibility :read} 8 | :remote {:store {:backend :jdbc 9 | :dbtype "postgresql" 10 | :host "aws-0-eu-central-1.pooler.supabase.com" 11 | :port 6543 12 | :dbname "postgres" 13 | :table #env ADH_POSTGRES_TABLE 14 | :user #env ADH_POSTGRES_USER 15 | :password #env ADH_POSTGRES_PWD 16 | :jdbcUrl "jdbc:postgresql://aws-0-eu-central-1.pooler.supabase.com:6543/postgres?pgbouncer=true&sslmode=require&prepareThreshold=0"} 17 | :schema-flexibility :read} 18 | 19 | :distributed {}} 20 | :chat {:all-entities-image "bot.svg" 21 | :entities [{:id "03174558-327a-47a4-93e1-5812955243b6" 22 | :name "Kunnskapsassistent" 23 | :image "kudos-logo.png" 24 | :docs-collection "PROD_kudos_docs" 25 | :chunks-collection "PROD_kudos_chunks" 26 | :phrases-collection "PROD_kudos_phrases" 27 | :phrase-gen-prompt "keyword-search" 28 | :promptRagQueryRelax "You have access to a search API that returns relevant documentation. 29 | Your task is to generate an array of up to 7 search queries that are relevant to the current state of the conversation provided below. 30 | Use a variation of related keywords and synonyms for the queries, trying to be as general as possible. 31 | Include as many queries as you can think of, including and excluding terms. For example, include queries like [\"keyword_1 keyword_2\", \"keyword_1\", \"keyword_2\"]. Be creative. The more queries you include, the more likely you are to find relevant results. 32 | 33 | {messages} 34 | 35 | ", 36 | :promptRagGenerate 37 | "Du kan se på meg som en kunnskapsassistent. Klar for å svare på spørsmål du har om offentlig sektor. 38 | Du har tilgang til: 39 | - Kudos databasen med mer enn 3000 kunnskaps- og styrings- dokumenter 40 | - Foreløpig har du kun årsrapporter, tildelingsbrev og evalueringer fra 2020 til og med 2023 og statusrapporter fra 2020-2024. 41 | - Vi har søkt med følgende filtere: 42 | Dokument type: [\"Årsrapport\" \"Tildelingsbrev\" \"Evaluering\" \"Statusrapport\"] 43 | Vi skal forsøke å gi deg mange relevante avsnitt fra dokumenter i databasen vår, men det kan hende at noen av de er ikke relevant for brukerens spørsmål. Hvis du ikke vet svaret, si at du ikke vet. 44 | 45 | 46 | {context} 47 | 48 | Spørsmål: {question} 49 | Gi det hjelpsomme svaret nedenfor. 50 | Hvis spørsmålet er uklart, be om presisering, før du gir endelig svar 51 | Hvis bruker spør hvilke dokumenter du har tilgang til, si at du har den nevnte dokumenttypen til statlige virksomheter fra 2020-2023 (2020-2024 hvis statusrapport). Si at du ikke kan ramse opp alle dokumentene, og spør hvilke virksomheter de vil utforske nærmere. 52 | Bruk Markdown formattering for å gjøre svaret oversiktlig og for bedre lesbarhet. 53 | Ta med kilden i parentes, kun hvis du har brukt en kilde fra de relevante kildene. 54 | Gi alltid ditt hjelpsomme svar på norsk, med mindre spørsmålet er på et annet språk. Hvis spørsmålet er på et annet språk, gi svaret på det språket. 55 | 56 | Hjelpsomt svar: 57 | " 58 | :reasoning-languages ["en" "no"] 59 | :prompt ""}]}} 60 | -------------------------------------------------------------------------------- /config/ka_next.edn: -------------------------------------------------------------------------------- 1 | {:db-env :remote 2 | :db {:mem {:store {:backend :mem :id "schemaless"} 3 | :schema-flexibility :read} 4 | :local {:store {:backend :file 5 | :path "./datahike-db" 6 | :id "schemaless"} 7 | :schema-flexibility :read} 8 | :remote {:store {:backend :jdbc 9 | :dbtype "postgresql" 10 | :host #env ADH_POSTGRES_HOST 11 | :port 6543 12 | :dbname "postgres" 13 | :table #env ADH_POSTGRES_TABLE 14 | :user #env ADH_POSTGRES_USER 15 | :password #env ADH_POSTGRES_PWD 16 | :jdbcUrl #env ADH_POSTGRES_URL} 17 | :schema-flexibility :read 18 | :keep-history? false 19 | ;; :allow-unsafe-config true ;; USE to allow for table name changes 20 | } 21 | 22 | :distributed {}} 23 | :chat {:all-entities-image "bot.svg" 24 | :entities [{:id "71e1adbe-2116-478d-92b8-40b10a612d7b" 25 | :name "Kunnskapsassistent - NEXT" 26 | :image "kudos-logo.png" 27 | :docs-collection "NEXT_kudos_docs" 28 | :chunks-collection "NEXT_kudos_chunks" 29 | :phrases-collection "NEXT_kudos_phrases" 30 | :phrase-gen-prompt "keyword-search" 31 | :promptRagQueryRelax "You have access to a search API that returns relevant documentation. 32 | Your task is to generate an array of up to 7 search queries that are relevant to the current state of the conversation provided below. 33 | Use a variation of related keywords and synonyms for the queries, trying to be as general as possible. 34 | Include as many queries as you can think of, including and excluding terms. For example, include queries like [\"keyword_1 keyword_2\", \"keyword_1\", \"keyword_2\"]. Be creative. The more queries you include, the more likely you are to find relevant results. 35 | 36 | {messages} 37 | 38 | ", 39 | :promptRagGenerate 40 | "Du kan se på meg som en kunnskapsassistent. Klar for å svare på spørsmål du har om offentlig sektor. 41 | Du har tilgang til: 42 | - Kudos databasen med mer enn 3000 kunnskaps- og styrings- dokumenter 43 | - Foreløpig har du kun årsrapporter, tildelingsbrev og evalueringer fra 2020 til og med 2023 og statusrapporter fra 2020-2024. 44 | - Vi har søkt med følgende filtere: 45 | Dokument type: [\"Årsrapport\" \"Tildelingsbrev\" \"Evaluering\" \"Statusrapport\"] 46 | Vi skal forsøke å gi deg mange relevante avsnitt fra dokumenter i databasen vår, men det kan hende at noen av de er ikke relevant for brukerens spørsmål. Hvis du ikke vet svaret, si at du ikke vet. 47 | 48 | 49 | {context} 50 | 51 | Spørsmål: {question} 52 | Gi det hjelpsomme svaret nedenfor. 53 | Hvis spørsmålet er uklart, be om presisering, før du gir endelig svar 54 | Hvis bruker spør hvilke dokumenter du har tilgang til, si at du har den nevnte dokumenttypen til statlige virksomheter fra 2020-2023 (2020-2024 hvis statusrapport). Si at du ikke kan ramse opp alle dokumentene, og spør hvilke virksomheter de vil utforske nærmere. 55 | Bruk Markdown formattering for å gjøre svaret oversiktlig og for bedre lesbarhet. 56 | Ta med kilden i parentes, kun hvis du har brukt en kilde fra de relevante kildene. 57 | Gi alltid ditt hjelpsomme svar på norsk, med mindre spørsmålet er på et annet språk. Hvis spørsmålet er på et annet språk, gi svaret på det språket. 58 | 59 | Hjelpsomt svar: 60 | " 61 | :reasoning-languages ["en" "no"] 62 | :prompt ""}]}} 63 | 64 | -------------------------------------------------------------------------------- /src/chat_app/debug.cljc: -------------------------------------------------------------------------------- 1 | (ns chat-app.debug 2 | (:require #?(:clj [datahike.api :as d]) 3 | [hyperfiddle.electric-dom2 :as dom] 4 | [hyperfiddle.electric :as e] 5 | #?(:clj [models.db :refer [conn]]) 6 | [hyperfiddle.electric-ui4 :as ui])) 7 | 8 | #?(:clj (defonce !call-open-ai? (atom true))) 9 | (defonce !debug? (atom false)) 10 | 11 | (e/defn TreeView [db-data] 12 | (e/client 13 | (dom/div (dom/props {:class "p-4 h-full overflow-auto"}) 14 | (let [!expanded-views (atom #{}) 15 | expanded-views (e/watch !expanded-views)] 16 | (e/for-by identity [[k v] db-data] 17 | (let [expanded? (contains? expanded-views k)] 18 | (dom/div (dom/props {:class "cursor-pointer"}) 19 | (dom/on "click" (e/fn [_] 20 | (if-not expanded? 21 | (swap! !expanded-views conj k) 22 | (swap! !expanded-views disj k)))) 23 | (dom/div (dom/props {:class "flex px-2 -mx-2 font-bold rounded"}) 24 | (dom/p (dom/props {:class "w-4"}) 25 | (dom/text (if expanded? "▼" "▶"))) 26 | (dom/p (dom/text k " (count " (count (filter #(not (= :db/txInstant (:a %))) v)) ")"))) 27 | (when expanded? 28 | (dom/div (dom/props {:class "pl-4"}) 29 | (e/for-by identity [[k v] (group-by :e v)] 30 | (e/for-by identity [{:keys [e a v t asserted]} (filter #(not (= :db/txInstant (:a %))) v)] 31 | ;; (let [{:keys [e a v t]} v]) 32 | (dom/div (dom/props {:class "flex gap-4"}) 33 | (dom/p (dom/props {:class "w-4"}) 34 | (dom/text e)) 35 | (dom/p (dom/props {:class "w-1/3"}) 36 | (dom/text a)) 37 | (dom/p (dom/props {:class "w-1/3 text-ellipsis overflow-hidden"}) 38 | (dom/text v)) 39 | (dom/p (dom/props {:class "w-8"}) 40 | (dom/text asserted)))))))))))))) 41 | 42 | 43 | (e/defn DBInspector [{:keys [!active-conversation !conversation-entity]}] 44 | (e/server 45 | (let [db (e/watch conn) 46 | group-by-tx (fn [results] (reduce (fn [acc [e a v tx asserted]] 47 | (update acc tx conj {:e e :a a :v v :asserted asserted})) 48 | {} 49 | results)) 50 | db-data (let [results (d/q '[:find ?e ?a ?v ?tx ?asserted 51 | :where 52 | [?e ?a ?v ?tx ?asserted]] (d/history db))] 53 | (reverse (sort (group-by-tx results))))] 54 | (e/client 55 | (dom/div (dom/props {:class "z-30 absolute top-0 right-0 h-48 h-full w-full bg-red-500 overflow-auto"}) ;w-1/2 56 | (dom/p (dom/text "Active conversation: " (e/watch !active-conversation))) 57 | (dom/p (dom/text "Conversation entity: " (e/watch !conversation-entity))) 58 | ;; (dom/p (dom/text "View main: " view-main)) 59 | ;; (dom/p (dom/text "Convo dragged: " convo-dragged)) 60 | ;; (dom/p (dom/text "Folder dragged to : " folder-dragged-to)) 61 | (ui/button (e/fn [] (e/server (swap! !call-open-ai? not))) 62 | (dom/props {:class "px-4 py-2 bg-white"}) 63 | (dom/text "Call OpenAI?: " (e/server (e/watch !call-open-ai?)))) 64 | (TreeView. db-data)))))) 65 | 66 | 67 | (e/defn DebugController [] 68 | (e/client 69 | (let [debug? (e/watch !debug?)] 70 | #_(ui/button 71 | (e/fn [] (swap! !debug? not)) 72 | (dom/props {:class (str "absolute top-0 right-0 z-10 px-4 py-2 rounded text-black" 73 | (if-not debug? 74 | " bg-slate-500" 75 | " bg-red-500"))}) 76 | (dom/p (dom/text "Debug: " debug?)))))) -------------------------------------------------------------------------------- /src/chat_app/filters.cljc: -------------------------------------------------------------------------------- 1 | (ns chat-app.filters 2 | (:require [hyperfiddle.electric :as e] 3 | [hyperfiddle.electric-dom2 :as dom] 4 | [hyperfiddle.electric-ui4 :as ui] 5 | #?(:clj [models.db :as db]))) 6 | 7 | ;; Utility functions 8 | (defn toggle [s k] 9 | (if (s k) 10 | (disj s k) 11 | (conj s k))) 12 | 13 | (defn typesense-field->ui-name [field] 14 | ({"type" "dokumenttyper" 15 | "orgs_short" "organisasjoner" 16 | "orgs_long" "organisasjoner" 17 | "owner_short" "eiere" 18 | "owner_long" "eiere" 19 | "publisher_short" "utgivere" 20 | "publisher_long" "utgivere" 21 | "recipient_short" "mottakere" 22 | "recipient_long" "mottakere" 23 | "source_published_year" "år publisert"} field field)) 24 | 25 | ;; Filter Components 26 | (e/defn FilterField [{:as x :keys [expanded? options field]} ToggleOption ToggleFieldExpanded? enabled?] 27 | (e/client 28 | (dom/div ;; NB: this extra div is required, otherwise the card 29 | ;; will stretch to the bottom of the parent 30 | ;; regardless of content height. 31 | (dom/div 32 | (dom/props {:class (str "mb-4 space-y-2 p-4 rounded-md shadow-md border " (if enabled? "bg-white" "bg-gray-300"))}) 33 | (dom/div 34 | (dom/div 35 | (dom/props {:class "font-medium text-gray-800 mb-2"}) 36 | (dom/text (str "Velg " (typesense-field->ui-name field)))) 37 | 38 | (ui/button ToggleFieldExpanded? 39 | (dom/props {:class "px-4 py-2 bg-blue-500 text-white rounded-md hover:bg-blue-600 focus:outline-none"}) 40 | (dom/text (str (count (filter :selected? options)) " valgt"))) 41 | (when expanded? 42 | (dom/div 43 | (dom/props {:class "flex flex-col items-start space-y-2 mt-1 max-h-48 overflow-y-scroll"}) 44 | (e/for [{:keys [selected? count value]} options] 45 | (ui/button (e/fn [] (ToggleOption. value)) 46 | (dom/span 47 | (dom/props {:class "grid grid-cols-[16px_1fr_auto] items-center gap-2 w-full text-left p-2 hover:bg-gray-100 rounded-md"}) 48 | (dom/img (dom/props {:class "w-[16px] h-[16px] flex-shrink-0" 49 | :src (if selected? 50 | "icons/checked_checkbox.svg" 51 | "icons/unchecked_checkbox.svg")})) 52 | (dom/span 53 | (dom/props {:class "text-gray-800 whitespace-wrap"}) 54 | (dom/text value)) 55 | (dom/span 56 | (dom/props {:class "text-gray-600 text-right"}) 57 | (dom/text (str "(" count ")"))))))))))))) 58 | 59 | (e/defn FilterMsg [msg enabled?] 60 | (e/client 61 | (let [mfilter (:message.filter/value msg)] 62 | (e/for [[idx field] (map vector (range) (mfilter :ui/fields))] 63 | (FilterField. field 64 | (e/fn ToggleOption [option] 65 | (if-not enabled? 66 | (e/client 67 | (js/alert "Filteret kan ikke endres etter oppfølgningspørsmål er sendt")) 68 | (e/server 69 | (e/offload 70 | #(db/set-message-filter 71 | db/conn 72 | (:db/id msg) 73 | (-> (update-in mfilter [:fields idx :selected-options] toggle option) 74 | (dissoc :ui/fields))))))) 75 | (e/fn ToggleFieldExpanded? [] 76 | (e/server 77 | (e/offload 78 | #(db/set-message-filter 79 | db/conn 80 | (:db/id msg) 81 | (-> (update-in mfilter [:fields idx :expanded?] not) 82 | (dissoc :ui/fields)))))) 83 | enabled?))))) 84 | -------------------------------------------------------------------------------- /src/chat_app/ui.cljc: -------------------------------------------------------------------------------- 1 | (ns chat-app.ui 2 | (:require [chat-app.debug :as debug] 3 | [chat-app.rhizome :as rhizome] 4 | [hyperfiddle.electric :as e] 5 | [hyperfiddle.electric-dom2 :as dom] 6 | [hyperfiddle.electric-ui4 :as ui])) 7 | 8 | ;; Shared UI state 9 | #?(:cljs (defonce !sidebar? (atom true))) 10 | #?(:cljs (defonce !view-main (atom :home))) 11 | #?(:cljs (defonce !view-main-prev (atom nil))) 12 | #?(:cljs (defonce view-main-watcher (add-watch !view-main :main-prev (fn [_k _r os _ns] 13 | (println "this is os: " os) 14 | (when-not (= os :settings)) 15 | (reset! !view-main-prev os))))) 16 | 17 | ;; UI Utility Functions 18 | #?(:cljs (defn observe-resize [node callback] 19 | (let [observed-element node 20 | resize-observer (js/ResizeObserver. 21 | (fn [entries] 22 | (doseq [entry entries] 23 | (let [content-rect (.-contentRect entry) 24 | rect (.getBoundingClientRect (.-target entry)) 25 | y (.-y rect) 26 | width (.-width content-rect) 27 | height (.-height content-rect)] 28 | (println "Resized" width height y) 29 | (callback width height y)))))] 30 | (.observe resize-observer observed-element) 31 | (fn [] (.disconnect resize-observer))))) 32 | 33 | ;; Layout Components 34 | (e/defn Topbar [conversation-entity] 35 | (e/client 36 | (let [sidebar? (e/watch !sidebar?)] 37 | (dom/div (dom/props {:class "sticky w-full top-0 h-14 z-10"}) 38 | (dom/div (dom/props {:class (str "flex justify-between gap-4 px-4 py-4" 39 | (if sidebar? 40 | " w-[260px] bg-slate-100" 41 | " w-max"))}) 42 | (ui/button 43 | (e/fn [] (reset! !sidebar? (not @!sidebar?))) 44 | (dom/img (dom/props {:class "w-6 h-6" 45 | :src (if-not sidebar? 46 | "icons/panel-left-open.svg" 47 | "icons/panel-left-close.svg")}))) 48 | #_(ui/button (e/fn [] (reset! !view-main :entity-selection)) 49 | (dom/img (dom/props {:class "w-6 h-6" 50 | :src "icons/square-pen.svg"})))) 51 | #_(dom/div (dom/props {:class "flex gap-4 py-4 items-center text-slate-500"}) 52 | (dom/p (dom/text (:name conversation-entity)))))))) 53 | 54 | 55 | (e/defn Home [] 56 | (e/client 57 | (dom/div 58 | (dom/props {:class "max-h-full overflow-x-hidden"}) 59 | (dom/div 60 | (dom/props {:class "h-[calc(100vh-10rem)] w-full flex flex-col justify-center items-center"}) 61 | (dom/div 62 | (dom/props {:class "flex flex-col items-center mx-auto max-w-3xl px-4"}) 63 | 64 | (dom/div 65 | (dom/props {:class "w-full max-w-[600px]"}) 66 | 67 | ;; Title 68 | (dom/div 69 | (dom/h1 70 | (dom/props {:class "text-2xl font-bold text-center text-gray-500"}) 71 | (dom/text "Presis og pålitelig innsikt,")) 72 | (dom/h1 73 | (dom/props {:class "text-2xl font-bold text-center text-gray-500 mb-6"}) 74 | (dom/text "skreddersydd for deg."))) 75 | 76 | ;; Subtitle 77 | (dom/p 78 | (dom/props {:class "text-center text-lg text-gray-500"}) 79 | (dom/text "Utforsk Kudos-dokumenter fra 2020-2023")) 80 | (dom/p 81 | (dom/props {:class "text-center text-lg text-gray-500"}) 82 | (dom/text "som tildelingsbrev, årsrapporter og evalueringer.")) 83 | (dom/p 84 | (dom/props {:class "text-center text-lg text-gray-500"}) 85 | (dom/text "Kjapt og enkelt.")))))))) 86 | 87 | -------------------------------------------------------------------------------- /src/chat_app/entities.cljc: -------------------------------------------------------------------------------- 1 | (ns chat-app.entities 2 | (:require [chat-app.ui :as ui] 3 | [chat-app.chat :as chat] 4 | [hyperfiddle.electric :as e] 5 | [hyperfiddle.electric-dom2 :as dom] 6 | [hyperfiddle.electric-ui4 :as ui4] 7 | #?(:clj [models.db :refer [update-config] :as db]))) 8 | 9 | ;; Entity state 10 | (e/def entities-cfg 11 | (e/server (:chat (e/watch db/!config)))) 12 | 13 | (e/defn PromptEditor [] 14 | (e/client 15 | (let [cfg entities-cfg 16 | entity (when cfg (-> cfg :entities first)) 17 | {:keys [id promptRagGenerate promptRagQueryRelax]} (or entity {}) 18 | !promptRagGenerate (atom promptRagGenerate) 19 | !promptRagQueryRelax (atom promptRagQueryRelax) 20 | promptGenerate (e/watch !promptRagGenerate) 21 | promptQueryRelax (e/watch !promptRagQueryRelax)] 22 | (if-not entity 23 | (dom/div (dom/text "Loading...")) 24 | (dom/div 25 | (dom/props {:class "m-2 h-full overflow-y-auto"}) 26 | (dom/h3 (dom/text "Query relax prompt:")) 27 | (dom/div 28 | (dom/textarea 29 | (dom/props {:value (or promptQueryRelax "") 30 | :style {:width "94%" 31 | :height "200px" 32 | :font-family "monospace" 33 | :margin "10px" 34 | :padding "10px" 35 | :border "1px solid #ccc" 36 | :border-radius "4px"} 37 | :placeholder "Loading config file..."}) 38 | (dom/on "change" (e/fn [e] 39 | (when-some [v (not-empty (.. e -target -value))] 40 | (reset! !promptRagQueryRelax v))))) 41 | 42 | (dom/h3 (dom/text "Generate prompt:")) 43 | (dom/textarea 44 | (dom/props {:value (or promptGenerate "") 45 | :style {:width "94%" 46 | :height "200px" 47 | :font-family "monospace" 48 | :margin "10px" 49 | :padding "10px" 50 | :border "1px solid #ccc" 51 | :border-radius "4px"} 52 | :placeholder "Loading config file..."}) 53 | (dom/on "change" (e/fn [e] 54 | (when-some [v (not-empty (.. e -target -value))] 55 | (reset! !promptRagGenerate v))))) 56 | (dom/div (dom/props {:class (str "bottom-3" " absolute right-0 w-full border-transparent bg-gradient-to-b from-transparent via-white to-white pt-6 md:pt-2")}) 57 | (dom/button 58 | (dom/on "click" 59 | (e/fn [_] 60 | (println "Saving prompts") 61 | (e/server 62 | (e/offload 63 | #(update-config 64 | {:id id 65 | :promptRagGenerate promptGenerate 66 | :promptRagQueryRelax promptQueryRelax})) 67 | nil))) 68 | (dom/props {:class "px-4 py-2 bg-blue-500 text-white rounded-md hover:bg-blue-600 focus:outline-none"}) 69 | (dom/text "Save changes"))))))))) 70 | 71 | (e/defn EntitySelector [] 72 | (e/client 73 | (let [EntityCard (e/fn [id title img-src] 74 | (ui4/button (e/fn [] 75 | (let [entity (some #(when (= (:id %) id) %) (:entities entities-cfg))] 76 | (when entity 77 | (reset! ui/!view-main :pre-conversation) 78 | (reset! chat/!conversation-entity entity)))) 79 | (dom/props {:class "flex flex-col gap-4 items-center 80 | bg-white rounded-lg p-4 shadow-md 81 | hover:shadow-lg transition-shadow duration-300 82 | w-64 h-80"}) 83 | (dom/img (dom/props {:class "w-32 h-32 object-cover rounded-full" 84 | :src img-src})) 85 | (dom/h2 (dom/props {:class "text-xl font-bold text-gray-800"}) 86 | (dom/text title))))] 87 | (dom/div (dom/props {:class "flex flex-col items-center justify-center h-full"}) 88 | (dom/h1 (dom/props {:class "text-3xl font-bold mb-8"}) 89 | (dom/text "Velg en assistent")) 90 | (dom/div (dom/props {:class "flex flex-wrap gap-8 justify-center"}) 91 | (e/for [entity (:entities entities-cfg)] 92 | (EntityCard. (:id entity) (:name entity) (:image entity)))))))) 93 | -------------------------------------------------------------------------------- /src/chat_app/auth_ui.cljc: -------------------------------------------------------------------------------- 1 | (ns chat-app.auth-ui 2 | (:require [chat-app.rhizome :as rhizome] 3 | [chat-app.webauthn :as webauthn] 4 | [hyperfiddle.electric :as e] 5 | [hyperfiddle.electric-dom2 :as dom] 6 | #?(:clj [chat-app.auth :as auth]) 7 | #?(:clj [models.db :refer [conn auth-conn fetch-user-id] :as db]))) 8 | 9 | ;; Client-side utility functions 10 | #?(:cljs (defn get-from-local-storage [key] 11 | (.getItem js/localStorage key))) 12 | 13 | (defonce !prepared-opts (atom nil)) 14 | 15 | ;; Authentication Components 16 | (e/defn HandleRegistration [] 17 | (e/client 18 | (println "this is a test") 19 | (let [create-opts (e/server 20 | (let [current-user (:user-id (auth/verify-token (get-in e/http-request [:cookies "auth-token" :value]))) 21 | session-id (get-in e/http-request [:headers "sec-websocket-key"])] 22 | (webauthn/create-public-key-options 23 | session-id 24 | {:name current-user 25 | :display-name current-user 26 | :rp-name "company-name" 27 | :rp-id "localhost"}))) 28 | cb #(reset! webauthn/!created-key %) 29 | prepared-opts (webauthn/prepare-for-creation create-opts)] 30 | (reset! !prepared-opts create-opts) 31 | (webauthn/create-credential prepared-opts cb) 32 | nil))) 33 | 34 | (e/defn AuthAdminDashboard [] 35 | (e/client 36 | (dom/div 37 | (dom/props {:class "max-h-full overflow-x-hidden"}) 38 | (dom/div 39 | (dom/props {:class "p-8 flex flex-col gap-4"}) 40 | (let [token (e/client (get-from-local-storage "auth-token"))] 41 | (dom/div 42 | (dom/p (dom/text "Local storage JWT: " token)) 43 | (dom/p (dom/text "Local storage JWT unsigned: " (e/server (auth/verify-token token)))) 44 | (let [http-cookie-jwt (e/server (get-in e/http-request [:cookies "auth-token" :value]))] 45 | (dom/p 46 | (dom/text 47 | "created-by: " 48 | (e/server 49 | (let [user-email (:user-id (auth/verify-token 50 | (get-in e/http-request [:cookies "auth-token" :value])))] 51 | (e/offload #(fetch-user-id user-email)))))) 52 | (dom/p (dom/text "HTTP Only cookie: " http-cookie-jwt)) 53 | (let [cookie (e/server (auth/verify-token http-cookie-jwt)) 54 | jwt-expiry (:expiry cookie)] 55 | (dom/p (dom/text "HTTP Only cookie: " cookie)) 56 | (dom/p (dom/text "HTTP Only cookie user: " (:user-id cookie))) 57 | (dom/p (dom/text "HTTP Only cookie expiry: " jwt-expiry)))))) 58 | 59 | (dom/button 60 | (dom/props {:class "px-4 py-2 bg-black text-white rounded"}) 61 | (dom/on "click" (e/fn [e] (set! (.-href js/window.location) "/logout"))) 62 | (dom/text "Sign out")) 63 | ;; 64 | (let [!email (atom nil) email (e/watch !email)] 65 | (dom/div 66 | (dom/props {:class "flex gap-4"}) 67 | (dom/input 68 | (dom/props {:class "px-4 py-2 border rounded" 69 | :placeholder "Enter email" 70 | :value email}) 71 | (dom/on "keyup" (e/fn [e] 72 | (if-some [v (not-empty (.. e -target -value))] 73 | (reset! !email v) 74 | (reset! !email nil))))) 75 | (dom/button (dom/props {:class "px-4 py-2 rounded bg-black hover:bg-slate-800 text-white"}) 76 | (dom/on "click" (e/fn [_] 77 | (when-let [email @!email] 78 | (e/server 79 | (auth/send-confirmation-code "kunnskap@digdir.cloud" (auth/generate-confirmation-code email)) 80 | nil)))) 81 | (dom/text "Send Code")) 82 | (dom/button 83 | (dom/props {:class "px-4 py-2 rounded bg-black hover:bg-slate-800 text-white"}) 84 | (dom/on "click" (e/fn [_] 85 | (when-let [email @!email] 86 | (e/server 87 | (let [current-user (:user-id (auth/verify-token (get-in e/http-request [:cookies "auth-token" :value]))) 88 | admin-id (e/offload #(fetch-user-id conn current-user))] 89 | (e/offload #(auth/create-new-user {:email email 90 | :creator-id admin-id})) 91 | nil) 92 | (auth/generate-confirmation-code email) 93 | nil)))) 94 | (dom/text "Generate Code")))) 95 | 96 | (dom/p (dom/text "Auth db")) 97 | (dom/pre (dom/text (rhizome/pretty-print (e/server (e/offload #(auth/all-accounts auth-conn)))))) 98 | 99 | 100 | (dom/p (dom/text "Active Confirmation Codes")) 101 | (dom/ul (dom/props {:class "flex flex-col gap-2"}) 102 | (e/for-by identity [user-code (e/server (map (fn [[key value]] [key (update value :expiry str)]) 103 | (e/watch auth/confirmation-codes)))] 104 | 105 | (dom/li (dom/props {:class "flex"}) 106 | (dom/p (dom/text user-code)) 107 | (dom/button 108 | (dom/props {:class "px-2 py-1 rounded bg-black text-white"}) 109 | (dom/on "click" (e/fn [_] 110 | (e/server 111 | (swap! auth/confirmation-codes dissoc (first user-code)) 112 | nil))) 113 | (dom/text "Remove"))))))))) 114 | -------------------------------------------------------------------------------- /src/chat_app/auth.clj: -------------------------------------------------------------------------------- 1 | (ns chat-app.auth 2 | (:require [buddy.sign.jwt :as jwt] 3 | [tick.core :as t] 4 | [clojure.string :as str] 5 | ;; [babashka.http-client :as http] 6 | [clj-http.client :as http2] 7 | [clojure.data.json :as json] 8 | [nano-id.core :refer [nano-id]] 9 | [datahike.api :as d] 10 | [hiccup2.core :as h] 11 | [models.db :as db :refer [cfg delayed-connection dh-schema]])) 12 | 13 | ;; ### Key Metrics to Track for Security 14 | ;; 1. **Failed login attempts**: Monitor for brute force attempts. 15 | ;; 2. **IP addresses**: Track unusual IPs or geolocation changes. 16 | ;; 3. **Session duration**: Detect abnormally long sessions. 17 | ;; 4. **Successful logins**: Correlate with user behavior or unusual patterns. 18 | ;; 5. **Password reset requests**: Frequent requests can indicate compromise. 19 | ;; 6. **Account creation activity**: Watch for bots or bulk account creation. 20 | ;; 7. **Access to admin areas**: Monitor elevated permissions usage. 21 | 22 | ;; ### Common Administration Functions 23 | ;; - [ ] **Create new account**: Adding users with roles. 24 | ;; - [ ] **Reset password**: Manual or user-initiated. 25 | ;; - [ ] **Disable account**: Temporary or permanent deactivation. 26 | ;; - [ ] **Update roles/permissions**: Adjust user access. 27 | 28 | 29 | ;; DB transactions 30 | (defn create-new-user [{:keys [email creator-id] :as data}] 31 | (let [conn @delayed-connection] 32 | (println "called create new user with: " email " and data " data) 33 | (if-not (d/entity @conn [:user/email email]) 34 | (let [user-id (nano-id)] 35 | (println "creating new user with: " email " and data " data) 36 | (d/transact conn [{:user/id user-id 37 | :user/email email 38 | :user/created (str (t/now)) 39 | :user/created-by (or creator-id user-id)}])) 40 | {:error "User already exists"}))) 41 | 42 | ;; DB queries 43 | (defn accounts-created-by [db email] 44 | (d/q '[:find (pull ?created-account [*]) 45 | :in $ ?creator-email 46 | :where 47 | [?creator :user/email ?creator-email] 48 | [?creator :user/id ?creator-id] 49 | [?created-account :user/created-by ?creator-id] 50 | [(not= ?created-account ?creator)]] 51 | db email)) 52 | 53 | (defn all-accounts [db] 54 | (d/q '[:find (pull ?e [*]) 55 | :where 56 | [?e :user/id] 57 | [?e :user/email] 58 | [?e :user/created-by]] 59 | db)) 60 | 61 | 62 | (def secret "cd5e984eaf41-16f2b708-9af6-497f") 63 | 64 | (defn admin-user? [email] 65 | (let [admins-env (or (System/getenv "ADMIN_USER_EMAILS") "") 66 | admins (set (map str/trim (str/split admins-env #" ")))] 67 | (contains? admins email))) 68 | 69 | (defn domain-whitelist [email] 70 | (let [domains (set (map str/trim (str/split (or (System/getenv "ALLOWED_DOMAINS") "") #" ")))] 71 | (contains? domains email))) 72 | 73 | (defn approved-domain? [email] 74 | (let [[_local-part domain] (str/split email #"@")] 75 | (boolean (domain-whitelist (str "@" domain))))) 76 | 77 | (def confirmation-codes (atom {})) 78 | 79 | (defn generate-confirmation-code [email] 80 | (let [confirmation-code (format "%06d" (rand-int 1000000))] 81 | (swap! confirmation-codes assoc email {:code confirmation-code 82 | :expiry (t/>> (t/now) (t/new-duration 10 :minutes))}) 83 | confirmation-code)) 84 | 85 | (defn valid-code? [email confirmation-code] 86 | (let [{:keys [code expiry]} (get @confirmation-codes email)] 87 | (and (t/< (t/now) expiry) (= code confirmation-code)))) 88 | 89 | (defn create-expiry [{:keys [multiplier timespan]}] 90 | (-> (t/now) 91 | (t/>> (t/new-duration multiplier timespan)) 92 | (t/inst) 93 | (.getTime))) 94 | 95 | (defn create-token [user-id expiry] 96 | (jwt/sign {:user-id user-id 97 | :expiry (/ expiry 1000)} secret)) 98 | 99 | (defn verify-token [token] 100 | (try 101 | (let [{:keys [user-id expiry]} (jwt/unsign token secret) 102 | current-time (/ (.getTime (t/inst (t/now))) 1000)] 103 | (if (< current-time expiry) 104 | {:valid true 105 | :user-id user-id 106 | :expiry (* expiry 1000)} 107 | {:valid false 108 | :reason "Token expired"})) 109 | (catch Exception e 110 | {:valid false :reason (str "Invalid token: " (.getMessage e))}))) 111 | 112 | ;; Confirmation code 113 | 114 | (def postmark-url "https://api.postmarkapp.com/email") 115 | (def email-server-token (or (System/getenv "POSTMARK_API_KEY") "")) 116 | 117 | 118 | (defn send-confirmation-code [from to code] 119 | (http2/post postmark-url 120 | {:headers {"Accept" "application/json" 121 | "Content-Type" "application/json" 122 | "X-Postmark-Server-Token" email-server-token} 123 | :body (json/write-str 124 | {:From from 125 | :To to 126 | :Subject (str "digdir.cloud - pålogging") 127 | :TextBody (str "For å gjennomføre påloggingen, skriv følgende kode i nettleseren: " 128 | code "\n\n" 129 | "Trenger du assistanse? Vennligst ta kontakt med oss på hjelp@digdir.cloud.") 130 | :HtmlBody (str (h/html 131 | [:html 132 | [:body 133 | [:p "For å gjennomføre påloggingen, skriv følgende kode i nettleseren: "] 134 | [:p {:style "font-size: 24px; font-weight: bold; color: #2D3748;"} code] 135 | [:p 136 | [:span "Trenger du assistanse? Vennligst ta kontakt med oss: "] 137 | [:a {:href "mailto:hjelp@digdir.cloud"} "hjelp@digdir.cloud"] 138 | ]]])) 139 | :MessageStream "outbound"})})) 140 | 141 | 142 | 143 | ;; Email Validation 144 | 145 | (defn starts-or-ends-with? [s s-check] 146 | (or (str/starts-with? s s-check) 147 | (str/ends-with? s s-check))) 148 | 149 | (defn consecutive-dots? [local] 150 | (re-find #"\.\." local)) 151 | 152 | (defn split-local-domain [email] 153 | (str/split email #"@")) 154 | 155 | (defn alphanumeric? [s] 156 | (boolean (re-find #"^[a-zA-Z0-9.-]+$" s))) 157 | 158 | (defn numeric-tld? [tld] 159 | (not (boolean (re-find #"[a-zA-Z]" tld)))) 160 | 161 | (defn validate-email? [email] 162 | (let [email-parts (split-local-domain email) 163 | errors (atom []) 164 | check (fn [pred? error-msg] (when pred? (swap! errors conj error-msg)))] 165 | 166 | (check (not (= 2 (count email-parts))) "Missing or too many @") 167 | (check (consecutive-dots? email) "Consecutive .") 168 | 169 | (when (= 2 (count email-parts)) 170 | (check (some #(or 171 | (starts-or-ends-with? % "-") 172 | (starts-or-ends-with? % ".")) 173 | email-parts) "- or . found at the beginning or end of parts of email") 174 | (check (some #(not (alphanumeric? %)) email-parts) "parts of the email are not alphanumeric") 175 | (let [domain-parts (str/split (second email-parts) #"\.") 176 | not-valid-domain? (or 177 | (not (<= 2 (count domain-parts))) 178 | (some str/blank? domain-parts)) 179 | tld (when-not not-valid-domain? (last domain-parts))] 180 | (check not-valid-domain? "domain is missing a part") 181 | (when-not not-valid-domain? 182 | (check (numeric-tld? tld) "numeric tld")))) 183 | 184 | @errors)) 185 | 186 | 187 | 188 | 189 | 190 | -------------------------------------------------------------------------------- /src/chat_app/chat.cljc: -------------------------------------------------------------------------------- 1 | (ns chat-app.chat 2 | (:require [chat-app.ui :as ui] 3 | [chat-app.rhizome :as rhizome] 4 | [hyperfiddle.electric :as e] 5 | [hyperfiddle.electric-dom2 :as dom] 6 | [hyperfiddle.electric-ui4 :as ui4] 7 | [nano-id.core :refer [nano-id]] 8 | [clojure.string :as str] 9 | [markdown.core :as md2] 10 | [chat-app.debug :as debug] 11 | [chat-app.filters :refer [FilterMsg]] 12 | #?(:clj [models.db :refer [conn] :as db]) 13 | #?(:clj [chat-app.rag :as rag]))) 14 | 15 | ;; Chat state 16 | #?(:cljs (defonce !active-conversation (atom nil))) 17 | #?(:cljs (defonce !conversation-entity (atom nil))) 18 | #?(:clj (defonce !stream-msgs (atom {}))) ;{:convo-id nil :content nil :streaming false} 19 | #?(:clj (defonce !wait? (atom false))) 20 | #?(:clj (defonce !stream? (atom false))) 21 | 22 | ;; Message Components 23 | (e/defn BotMsg [msg-map] 24 | (e/client 25 | (let [conversation-entity (e/watch !conversation-entity) 26 | {:message/keys [created id text role kind voice]} msg-map 27 | {:keys [name image]} conversation-entity] 28 | (dom/div (dom/props {:class "flex w-full flex-col items-start"}) 29 | (dom/div (dom/props {:class "flex"}) 30 | (dom/img (dom/props {:class "rounded-full w-8 h-8" 31 | :src image})) 32 | (dom/div (dom/props {:class "prose whitespace-pre-wrap px-4 pt-1 max-w-[600px]"}) 33 | (case kind 34 | :kind/html (set! (.-innerHTML dom/node) text) 35 | :kind/markdown (set! (.-innerHTML dom/node) (md2/md-to-html-string text)) 36 | (set! (.-innerHTML dom/node) (md2/md-to-html-string text))))))))) 37 | 38 | (e/defn UserMsg [msg] 39 | (e/client 40 | (dom/div (dom/props {:class "flex w-full flex-col items-start"}) 41 | (let [msg-hovered? (dom/Hovered?.)] 42 | (dom/div (dom/props {:class "relative max-w-[70%] rounded-3xl bg-[#b8e9f8] px-5 py-2.5"}) 43 | ;; disabling edit button for now, should make it configurable 44 | #_(ui4/button (e/fn []) 45 | (dom/props {:class (str "absolute -left-12 top-1 hover:bg-[#f4f4f4] rounded-full flex justify-center items-center w-8 h-8" 46 | (if-not msg-hovered? 47 | " invisible" 48 | " visible")) 49 | :title "Edit message"}) 50 | (dom/img (dom/props {:class "w-4" :src "icons/pencil.svg"}))) 51 | (dom/p (dom/text msg))))))) 52 | 53 | (e/defn ResponseState [state] 54 | (e/client (dom/div (dom/props {:class "w-full"}) 55 | (dom/div 56 | (dom/props {:class "mx-auto flex flex-1 gap-4 text-base md:gap-5 lg:gap-6 md:max-w-3xl lg:max-w-[40rem] xl:max-w-[48rem]"}) 57 | (dom/div (dom/props {:class "flex w-full flex-col items-start"}) 58 | (let [msg-hovered? (dom/Hovered?.)] 59 | (dom/div (dom/props {:class "flex"}) 60 | (dom/img (dom/props {:class "w-8 h-8" 61 | :src "icons/progress-circle.svg"})) 62 | (dom/div (dom/props {:class "prose whitespace-pre-wrap px-4 pt-1 max-w-[600px]"}) 63 | (dom/text state))))))))) 64 | 65 | (e/defn RenderMsg [msg-map last?] 66 | (e/client 67 | (let [{:message/keys [created id text role kind voice]} msg-map 68 | _ (prn "message id" id "voice:" voice " kind: " kind)] 69 | (dom/div (dom/props {:class "w-full"}) 70 | (dom/div 71 | (dom/props {:class "mx-auto flex flex-1 gap-4 text-base md:gap-5 lg:gap-6 md:max-w-3xl lg:max-w-[40rem] xl:max-w-[48rem]"}) 72 | (case voice 73 | :user (UserMsg. text) 74 | :assistant (BotMsg. msg-map) 75 | :filter (FilterMsg. msg-map last?) 76 | :agent (dom/div (dom/text)) 77 | :system (dom/div (dom/props {:class "group md:px-4 border-b border-black/10 bg-white text-gray-800"}) 78 | (dom/div (dom/props {:class "relative m-auto flex p-4 text-base md:max-w-2xl md:gap-6 md:py-6 lg:max-w-2xl lg:px-0 xl:max-w-3xl"}) 79 | #_(dom/div (dom/props {:class "min-w-[40px] text-right font-bold"}) 80 | (set! (.-innerHTML dom/node) bot-icon)) 81 | (dom/div (dom/props {:class "prose whitespace-pre-wrap flex-1"}) 82 | (set! (.-innerHTML dom/node) (md2/md->html text))) 83 | #_(dom/div (dom/props {:class "md:-mr-8 ml-1 md:ml-0 flex flex-col md:flex-row gap-4 md:gap-1 items-center md:items-start justify-end md:justify-start"}) 84 | (dom/button (dom/props {:class "invisible group-hover:visible focus:visible text-gray-500 hover:text-gray-700 dark:text-gray-400 dark:hover:text-gray-300"}) 85 | 86 | (set! (.-innerHTML dom/node) delete-icon))))) 87 | )))))) 88 | 89 | (e/defn HandleChatMsg [user-query last-message-is-filter?] 90 | (e/client 91 | (let [current-convo-id (e/watch !active-conversation) 92 | conversation-entity (e/watch !conversation-entity) 93 | ;; initialize new conversation id on the client, as it wasn't possible to get 94 | ;; a server-side generated id back to the client reliably 95 | new-convo-id (nano-id)] 96 | (e/server 97 | (let [new-convo-id-srv new-convo-id ;; workaround for electric v2 bug 98 | call-open-ai? (e/watch debug/!call-open-ai?) 99 | stream? (e/watch !stream?) 100 | base-msg-data {:stream? stream? 101 | :call-open-ai? call-open-ai? 102 | :user-query user-query 103 | :conversation-entity conversation-entity} 104 | msg-data (if current-convo-id 105 | (assoc base-msg-data :convo-id current-convo-id) 106 | (assoc base-msg-data :convo-id new-convo-id-srv :new-convo? true)) 107 | job-data {:type (if last-message-is-filter? :new :followup) 108 | :msg-data msg-data}] 109 | 110 | ;; Enqueue the job instead of running it immediately 111 | (rag/enqueue-rag-job job-data) 112 | 113 | (e/client 114 | (println "current-convo-id: " current-convo-id "new-convo-id: " new-convo-id) 115 | (when-not current-convo-id 116 | (reset! !active-conversation new-convo-id) 117 | (reset! ui/!view-main :conversation)))) 118 | nil)))) 119 | 120 | (e/defn PromptInput [{:keys [convo-id messages]}] 121 | (e/client 122 | ;; TODO: add the system prompt to the message list 123 | (let [!input-node (atom nil) 124 | wait? (e/server (e/watch !wait?))] 125 | (dom/div (dom/props {:class (str (if (rhizome/mobile-device?) "bottom-8" "bottom-0") " absolute left-0 w-full border-transparent bg-gradient-to-b from-transparent via-white to-white pt-6 md:pt-1")}) 126 | (dom/div (dom/props {:class "stretch mx-2 mt-4 flex flex-row gap-3 last:mb-2 md:mx-4 md:mt-[52px] md:last:mb-6 lg:mx-auto lg:max-w-3xl"}) 127 | (dom/div (dom/props {:class "flex flex-col w-full gap-2"}) 128 | (dom/div (dom/props {:class "relative flex w-full flex-grow flex-col rounded-md border border-black/10 bg-white shadow-[0_0_10px_rgba(0,0,0,0.10)] sm:mx-4"}) 129 | (dom/textarea 130 | (dom/props {:id "prompt-input" 131 | :class "min-h-[44px] max-h-[200px] m-0 w-full resize-none border-0 bg-transparent p-0 py-2 pr-8 pl-10 text-black md:py-3 md:pl-10 overflow-y-auto" 132 | :placeholder (str "Hva kan jeg hjelpe deg med?") 133 | :value "" 134 | :rows "4" 135 | :disabled wait?}) 136 | (reset! !input-node dom/node) 137 | (.focus dom/node) 138 | (dom/on "keydown" 139 | (e/fn [e] 140 | (when (= "Enter" (.-key e)) 141 | ;; Only submit message if shift key is not pressed 142 | (if (.-shiftKey e) 143 | ;; Allow shift+enter to create a new line 144 | nil 145 | ;; Regular enter submits the message 146 | (do 147 | (.preventDefault e) 148 | (when-some [v (not-empty (.. e -target -value))] 149 | (when-not (str/blank? v) 150 | (HandleChatMsg. v 151 | (-> (last messages) :message.filter/value boolean)))) 152 | (set! (.-value @!input-node) ""))))))) 153 | (let [wait? (e/server (e/watch !wait?))] 154 | (ui4/button 155 | (e/fn [] 156 | (when-some [v (not-empty (.-value @!input-node))] 157 | (when-not (str/blank? v) 158 | (HandleChatMsg. v 159 | (-> (last messages) :message.filter/value boolean)))) 160 | (set! (.-value @!input-node) "")) 161 | (dom/props {:title (when wait? "Functionality not implemented") ;TODO: implement functionality to stop process 162 | :class "absolute right-2 top-2 rounded-sm p-1 text-neutral-800 opacity-60 hover:bg-neutral-200 hover:text-neutral-900"}) 163 | (if-not wait? 164 | (dom/img (dom/props {:src "icons/old/send.svg"})) 165 | (dom/img (dom/props {:src "icons/circle-stop.svg"})))))) 166 | 167 | ;; Disclaimer text space below the input field 168 | (dom/div (dom/props {:class "mt-3 px-4 text-xs text-gray-500 text-center"}) 169 | (dom/text "​Kunnskapsassistenten kan gjøre feil. Husk å sjekke viktig informasjon.")))))))) 170 | 171 | (e/defn Conversation [] 172 | (e/server 173 | (let [db (e/watch conn)] 174 | (e/client 175 | (let [convo-id (e/watch !active-conversation) 176 | conversation-entity (e/watch !conversation-entity) 177 | {:keys [prompt image full-name name]} conversation-entity 178 | response-states (e/server (e/watch rag/!response-states)) 179 | !chat-container (atom nil) 180 | !is-at-bottom (atom false) 181 | scroll-to-bottom (fn [] 182 | (when-let [container @!chat-container] 183 | (when @!is-at-bottom 184 | (println "Scrolling to bottom") 185 | (set! (.-scrollTop container) (.-scrollHeight container)))))] 186 | (dom/div 187 | (dom/props {:class "max-h-full overflow-x-hidden"}) 188 | (reset! !chat-container dom/node) 189 | (dom/on "scroll" (e/fn [e] 190 | (let [target (.-target e) 191 | scroll-bottom (+ (.-scrollTop target) (.-clientHeight target)) 192 | scroll-height (.-scrollHeight target) 193 | at-bottom (<= (- scroll-height scroll-bottom) 5.0)] 194 | (when at-bottom 195 | (println "Scrolling, at bottom?" at-bottom "scroll values - bottom:" scroll-bottom 196 | "height:" scroll-height "diff:" (- scroll-height scroll-bottom))) 197 | (reset! !is-at-bottom at-bottom)))) 198 | 199 | (when 200 | (and (some? convo-id) 201 | (some? conversation-entity)) 202 | (let [messages (e/server (e/offload #(rag/prepare-conversation db convo-id conversation-entity)))] 203 | (dom/div 204 | (dom/props {:class "max-h-full overflow-x-hidden"}) 205 | (ui/observe-resize dom/node scroll-to-bottom) 206 | 207 | (dom/div 208 | (dom/props {:class "flex flex-col stretch justify-center items-center h-full lg:max-w-3xl mx-auto gap-4 pb-[110px]"}) 209 | 210 | (dom/div (dom/props {:class "flex flex-col gap-8 items-center"}) 211 | 212 | #_(dom/img (dom/props {:class "w-48 mx-auto rounded-full" 213 | :src image})) 214 | (dom/h1 (dom/props {:class "text-2xl"}) (dom/text (or full-name name)))) 215 | (when messages ;todo: check if this is still needed 216 | (e/for [msg (butlast messages)] 217 | (RenderMsg. msg false)) 218 | (RenderMsg. (last messages) true) 219 | (when-let [rs (first response-states)] (ResponseState. rs))) 220 | 221 | (let [stream-msgs (e/server (e/watch !stream-msgs))] 222 | (when (:streaming (get stream-msgs convo-id)) 223 | (when-let [content (:content (get stream-msgs convo-id))] 224 | (BotMsg. content)))) 225 | 226 | (PromptInput. {:convo-id convo-id 227 | :messages messages}))))))))))) 228 | 229 | -------------------------------------------------------------------------------- /src-dev/user.clj: -------------------------------------------------------------------------------- 1 | (ns user 2 | (:require [dev] 3 | [datahike.api :as d] 4 | [datahike-jdbc.core] 5 | [chat-app.main :as main] 6 | [chat-app.auth :as auth] 7 | [portal.api :as p])) 8 | 9 | 10 | (comment 11 | ;; here is where we can do stuff in the portal repl 12 | (def portal (p/open)) 13 | (add-tap #'p/submit) 14 | 15 | (def db (d/db main/conn)) 16 | 17 | (defn latest-convo-id [] 18 | (d/q '[:find ?convo-id])) 19 | 20 | (defn newest-convo-id [] 21 | (let [results (d/q '[:find ?convo-id ?created 22 | :where 23 | [?e :conversation/id ?convo-id] 24 | [?e :conversation/created ?created]] 25 | db) 26 | sorted-results (sort-by second > results)] 27 | (when (seq sorted-results) 28 | (first (first sorted-results))))) 29 | 30 | (newest-convo-id) 31 | 32 | (defn fetch-messages-for-newest-convo [] 33 | (let [newest-id (newest-convo-id)] 34 | (when newest-id 35 | (main/fetch-convo-messages newest-id)))) 36 | 37 | (tap> (fetch-messages-for-newest-convo)) 38 | (def convo-id (:convo-id (newest-convo-id))) 39 | 40 | (newest-convo-id) 41 | 42 | (def all-messages (main/fetch-convo-messages db convo-id)) 43 | (defn agent-messages [convo-id] (main/fetch-convo-messages db convo-id :agent)) 44 | (defn user-messages [convo-id] (main/fetch-convo-messages db convo-id :user)) 45 | 46 | (tap> (user/agent-messages convo-id)) 47 | (tap> (user/user-messages convo-id)) 48 | ) 49 | 50 | ;; auth tests 51 | 52 | (comment 53 | 54 | (auth/generate-confirmation-code "wd@itonomi.com") 55 | 56 | @confirmation-codes 57 | (def token (auth/create-token "wd@itonomi.com" (auth/create-expiry {:multiplier 5 58 | :timespan :seconds}))) 59 | (auth/verify-token token) 60 | 61 | 62 | ;; Email validation 63 | (auth/validate-email? "examplemydomaincom") 64 | (auth/validate-email? "example@test@my-domain.com") 65 | (auth/validate-email? "example.@my-domain.com") 66 | (auth/validate-email? "example@com") 67 | (auth/validate-email? "example..test@my-domain.com") 68 | (auth/validate-email? "-example@my-domain.com") 69 | (auth/validate-email? "exam&ple@my-domain.com") 70 | (auth/validate-email? "example@my-domain.com.123") 71 | 72 | (auth/send-confirmation-code "bdb@itonomi.com" "benjamin@bdbrodie.com" "8983")) 73 | 74 | 75 | ;; rag tests 76 | 77 | (def kudos-params 78 | {:translated_user_query "Hva har vegvesenet gjort innen innovasjon?" 79 | :original_user_query "Hva har vegvesenet gjort innen innovasjon?" 80 | :user_query_language_name "Norwegian" 81 | :promptRagQueryRelax "You have access to a search API that returns relevant documentation.\nYour task is to generate an array of up to 7 search queries that are relevant to this question. Use a variation of related keywords and synonyms for the queries, trying to be as general as possible.\nInclude as many queries as you can think of, including and excluding terms. For example, include queries like ['keyword_1 keyword_2', 'keyword_1', 'keyword_2']. Be creative. The more queries you include, the more likely you are to find relevant results.\n", 82 | :promptRagGenerate "Use the following pieces of information to answer the user's question.\nIf you don't know the answer, just say that you don't know, don't try to make up an answer.\n\nContext: {context}\n\nQuestion: {question}\n\nOnly return the helpful answer below, using Markdown for improved readability.\n\nHelpful answer:\n", 83 | :phrase-gen-prompt "keyword-search" 84 | :maxSourceDocCount 10 85 | :maxContextLength 10000 86 | :maxSourceLength 40000 87 | :docsCollectionName "DEV_kudos-docs" 88 | :chunksCollectionName "DEV_kudos-chunks" 89 | :phrasesCollectionName "DEV_kudos-phrases" 90 | :stream_callback_msg1 nil 91 | :stream_callback_msg2 nil 92 | :streamCallbackFreqSec 2.0 93 | :maxResponseTokenCount nil}) 94 | 95 | 96 | (def studio-params 97 | {:translated_user_query "Can you translate the resource texts to \"nynorsk\"?" 98 | :original_user_query "Kan du oversette ressurstekstene til nynorsk?" 99 | :user_query_language_name "Norwegian" 100 | :promptRagQueryRelax "You have access to a search API that returns relevant documentation.\nYour task is to generate an array of up to 7 search queries that are relevant to this question. Use a variation of related keywords and synonyms for the queries, trying to be as general as possible.\nInclude as many queries as you can think of, including and excluding terms. For example, include queries like ['keyword_1 keyword_2', 'keyword_1', 'keyword_2']. Be creative. The more queries you include, the more likely you are to find relevant results.\n", 101 | :promptRagGenerate "Use the following pieces of information to answer the user's question.\nIf you don't know the answer, just say that you don't know, don't try to make up an answer.\n\nContext: {context}\n\nQuestion: {question}\n\nOnly return the helpful answer below.\n\nHelpful answer:\n", 102 | :phrase-gen-prompt "keyword-search" 103 | :maxSourceDocCount 10 104 | :maxContextLength 10000 105 | :maxSourceLength 40000 106 | :docsCollectionName "DEV_studio-docs" 107 | :chunksCollectionName "DEV_studio-chunks" 108 | :phrasesCollectionName "DEV_studio-phrases" 109 | :stream_callback_msg1 nil 110 | :stream_callback_msg2 nil 111 | :streamCallbackFreqSec 2.0 112 | :maxResponseTokenCount nil}) 113 | 114 | 115 | (def altinn-entity-id "ec8c8be0-587d-4269-804a-78ce493801b5") 116 | 117 | 118 | (def kudos-entity-id "71e1adbe-2116-478d-92b8-40b10a612d7b") 119 | 120 | (def ai-guide-entity-id "7i8dadbe-0101-f0e1-92b8-40b10a61cdcd") 121 | 122 | 123 | (comment 124 | 125 | (def conn @delayed-connection) 126 | 127 | (def ai-rag-params {:conversation-id (nano-id) 128 | :entity-id ai-guide-entity-id 129 | :translated_user_query "hvilke kategorier gjelder for KI system risiko?" 130 | :original_user_query "hvilke kategorier gjelder for KI system risiko?" 131 | :user_query_language_name "Norwegian" 132 | :promptRagQueryRelax "You have access to a search API that returns relevant documentation.\nYour task is to generate an array of up to 7 search queries that are relevant to this question. Use a variation of related keywords and synonyms for the queries, trying to be as general as possible.\nInclude as many queries as you can think of, including and excluding terms. For example, include queries like ['keyword_1 keyword_2', 'keyword_1', 'keyword_2']. Be creative. The more queries you include, the more likely you are to find relevant results.\n", 133 | :promptRagGenerate "Use the following pieces of information to answer the user's question.\nIf you don't know the answer, just say that you don't know, don't try to make up an answer.\n\nContext: {context}\n\nQuestion: {question}\n\nOnly return the helpful answer below.\n\nHelpful answer:\n", 134 | :phrase-gen-prompt "keyword-search" 135 | :maxSourceDocCount 10 136 | :maxContextLength 10000 137 | :maxSourceLength 40000 138 | :docsCollectionName "AI-GUIDE_docs_2024-10-28" 139 | :chunksCollectionName "AI-GUIDE_chunks_2024-10-28" 140 | :phrasesCollectionName "AI-GUIDE_phrases_2024-10-28" 141 | :stream_callback_msg1 nil 142 | :stream_callback_msg2 nil 143 | :streamCallbackFreqSec 2.0 144 | :maxResponseTokenCount nil}) 145 | 146 | (def kudos-rag-params {:conversation-id (nano-id) 147 | :entity-id kudos-entity-id 148 | :translated_user_query "DSB 2022" 149 | :original_user_query "DSB 2022" 150 | :user_query_language_name "Norwegian" 151 | :promptRagQueryRelax "You have access to a search API that returns relevant documentation.\nYour task is to generate an array of up to 7 search queries that are relevant to this question. Use a variation of related keywords and synonyms for the queries, trying to be as general as possible.\nInclude as many queries as you can think of, including and excluding terms. For example, include queries like ['keyword_1 keyword_2', 'keyword_1', 'keyword_2']. Be creative. The more queries you include, the more likely you are to find relevant results.\n", 152 | :promptRagGenerate "Use the following pieces of information to answer the user's question.\nIf you don't know the answer, just say that you don't know, don't try to make up an answer.\n\nContext: {context}\n\nQuestion: {question}\n\nOnly return the helpful answer below.\n\nHelpful answer:\n", 153 | :phrase-gen-prompt "keyword-search" 154 | :filter {:type :typesense 155 | :fields [{:type :multiselect 156 | :selected-options #{"Tildelingsbrev"} 157 | :field "type"}]} 158 | :maxSourceDocCount 10 159 | :maxContextLength 10000 160 | :maxSourceLength 40000 161 | :docsCollectionName "TEST_kudos_docs" 162 | :chunksCollectionName "TEST_kudos_chunks" 163 | :phrasesCollectionName "TEST_kudos_phrases" 164 | :stream_callback_msg1 nil 165 | :stream_callback_msg2 nil 166 | :streamCallbackFreqSec 2.0 167 | :maxResponseTokenCount nil}) 168 | 169 | (def rag-results (rag-pipeline kudos-rag-params conn)) 170 | 171 | ;; (answer-followup-user-query conn (:conversation-id rag-results) "hvilke kriterie definerer høyrisiko?") 172 | 173 | (defn get-newest-conversation-id 174 | "Retrieves the ID of the most recently created conversation." 175 | [] 176 | (let [query '[:find (max ?created) ?id 177 | :where 178 | [?e :conversation/id ?id] 179 | [?e :conversation/created ?created]] 180 | result (d/q query @conn)] 181 | (when (seq result) 182 | (second (first result))))) 183 | 184 | (defn get-recent-conversations 185 | "Retrieves the 10 most recent conversations with their metadata. 186 | Returns a sequence of maps containing :id, :created, :topic" 187 | [] 188 | (let [query '[:find ?id (max ?created) ?topic 189 | :where 190 | [?e :conversation/id ?id] 191 | [?e :conversation/created ?created] 192 | [?e :conversation/topic ?topic] 193 | :limit 10] 194 | results (d/q query @conn)] 195 | (when (seq results) 196 | (->> results 197 | (map (fn [[id created topic]] 198 | {:id id 199 | :created created 200 | :topic topic})) 201 | (sort-by :created >))))) 202 | 203 | (get-recent-conversations) 204 | 205 | (def rag-params (assoc kudos-rag-params :conversation-id "7zvrhkUE_3I2sEoSBiJ16")) 206 | 207 | (def rag-params (assoc kudos-rag-params :conversation-id (get-newest-conversation-id))) 208 | (def durations {:total 0 209 | :analyze 0 210 | :generate_searches 0 211 | :execute_searches 0 212 | :phrase_similarity_search 0 213 | :colbert_rerank 0 214 | :rag_query 0 215 | :translation 0}) 216 | 217 | 218 | ;; RAG stuff 219 | 220 | 221 | (def convo-id (:conversation-id rag-params)) 222 | (db/transact-new-msg-thread conn {:convo-id convo-id 223 | :user-query (:original_user_query rag-params) 224 | :entity-id (:entity-id rag-params)}) 225 | 226 | (def extract-search-queries (query-relaxation (:translated_user_query rag-params) 227 | (:promptRagQueryRelax rag-params))) 228 | ;; durations (assoc durations :generate_searches (- (System/currentTimeMillis) start)) 229 | 230 | (def search-phrase-hits (lookup-search-phrases-similar 231 | (:phrasesCollectionName rag-params) 232 | (:docsCollectionName rag-params) 233 | extract-search-queries 234 | (:phrase-gen-prompt rag-params) 235 | (:filter rag-params))) 236 | (def retrieved-chunks (retrieve-chunks-by-id 237 | (:docsCollectionName rag-params) 238 | (:chunksCollectionName rag-params) 239 | search-phrase-hits)) 240 | 241 | ;; debug retrieve-chunks-by-id 242 | ;; seems to be a bug in Typesense. 243 | ;; - Iif you recreate the docs collection, 244 | ;; the chunks and phrases connection will fail silently in this query: 245 | (ts-client/multi-search ts-cfg 246 | {:searches 247 | (vector {:collection "TEST_kudos_chunks", :q "26803-1", 248 | :include_fields "id,chunk_id,doc_num,$TEST_kudos_docs(title,source_document_url)", 249 | :filter_by "chunk_id:=`26803-1`", 250 | :page 1, :per_page 1}), 251 | :limit_multi_searches 40} 252 | {:query_by "chunk_id"}) 253 | 254 | 255 | (retrieve-chunks-by-id "KUDOS_docs_2024-12-10" "KUDOS_chunks_2024-12-10" (vector {:chunk_id "26803-1", :rank 0.699999988079071, :index 0})) 256 | 257 | (count retrieved-chunks) 258 | 259 | (def reranked-results (rerank-chunks retrieved-chunks rag-params)) 260 | 261 | rag-params 262 | 263 | (def new-system-message 264 | (d/transact conn [{:conversation/id (:conversation-id rag-params) 265 | :conversation/messages [{:message/id (nano-id) 266 | :message/text formatted-message 267 | :message/role :system 268 | :message/voice :assistant 269 | :message/created (System/currentTimeMillis)}]}])) 270 | 271 | (def rag-params (assoc rag-params :conversation-id (get-newest-conversation-id))) 272 | 273 | (rag-pipeline rag-params conn) 274 | 275 | (defn get-conversation-messages 276 | "Retrieves all messages for the specified conversation ID." 277 | [conn conversation-id] 278 | (let [query '[:find ?text ?role ?voice ?created 279 | :in $ ?conv-id 280 | :where 281 | [?e :conversation/id ?conv-id] 282 | [?e :conversation/messages ?m] 283 | [?m :message/text ?text] 284 | [?m :message/role ?role] 285 | [?m :message/voice ?voice] 286 | [?m :message/created ?created]] 287 | results (d/q query @conn conversation-id)] 288 | (->> results 289 | (map (fn [[text role voice created]] 290 | {:text text 291 | :role role 292 | :voice voice 293 | :created created})) 294 | (sort-by :created)))) 295 | 296 | (get-conversation-messages conn "7zvrhkUE_3I2sEoSBiJ16") 297 | 298 | 299 | ;; 300 | ) 301 | -------------------------------------------------------------------------------- /src/chat_app/rag_test.cljc: -------------------------------------------------------------------------------- 1 | (ns chat-app.rag-test 2 | (:require [hyperfiddle.rcf :as rcf :refer [tests]] 3 | #?(:clj [chat-app.rag :as rag]) 4 | #?(:clj [typesense.client :as ts-client]))) 5 | 6 | #?(:clj (rcf/enable!)) 7 | 8 | #?(:clj 9 | (tests "facet-result->ui-field" 10 | ; converts facet result to UI field with no selected options 11 | (let [filter-field {:field "category" 12 | :selected-options #{}} 13 | options [{:count 5 :value "docs"} 14 | {:count 3 :value "articles"}] 15 | result (rag/facet-result->ui-field filter-field options)] 16 | (= {:field "category" 17 | :selected-options #{} 18 | :options [{:count 5 19 | :value "docs" 20 | :selected? false} 21 | {:count 3 22 | :value "articles" 23 | :selected? false}]} 24 | result)) 25 | 26 | ; converts facet result to UI field with selected options 27 | (let [filter-field {:field "category" 28 | :selected-options #{"docs"}} 29 | options [{:count 5 :value "docs"} 30 | {:count 3 :value "articles"}] 31 | result (rag/facet-result->ui-field filter-field options)] 32 | (= {:field "category" 33 | :selected-options #{"docs"} 34 | :options [{:count 5 35 | :value "docs" 36 | :selected? true} 37 | {:count 3 38 | :value "articles" 39 | :selected? false}]} 40 | result)))) 41 | 42 | #?(:clj 43 | (tests "fetch-facets" 44 | ; handles successful facet fetch 45 | (let [conversation-entity {:docs-collection "test-collection"} 46 | filter-map {:fields [{:field "category" 47 | :selected-options #{"docs"}}]} 48 | mock-results {:results [{:facet_counts [{:field_name "category" 49 | :stats {:count 5} 50 | :counts [{:count 5 :value "docs"} 51 | {:count 3 :value "articles"}]}]}]}] 52 | (with-redefs [ts-client/multi-search (constantly mock-results)] 53 | (let [result (rag/fetch-facets conversation-entity filter-map)] 54 | (= {:fields [{:field "category" 55 | :selected-options #{"docs"} 56 | :options [{:count 5 57 | :value "docs" 58 | :selected? true} 59 | {:count 3 60 | :value "articles" 61 | :selected? false}]}]} 62 | result)))) 63 | 64 | ; handles errors gracefully 65 | (let [conversation-entity {:docs-collection "test-collection"} 66 | filter-map {:fields [{:field "category" 67 | :selected-options #{"docs"}}]}] 68 | (with-redefs [ts-client/multi-search (fn [& _] (throw (Exception. "Test error")))] 69 | (let [result (rag/fetch-facets conversation-entity filter-map)] 70 | (= filter-map result)))))) 71 | 72 | 73 | (comment 74 | 75 | 76 | 77 | (def test-facet-results 78 | {:results 79 | [{:facet_counts 80 | [{:counts 81 | [{:count 2021, :highlighted "Tildelingsbrev", :value "Tildelingsbrev"} 82 | {:count 788, :highlighted "Årsrapport", :value "Årsrapport"} 83 | {:count 584, :highlighted "Evaluering", :value "Evaluering"}], 84 | :field_name "type", 85 | :sampled false, 86 | :stats {:total_values 3}} 87 | {:counts 88 | [{:count 394, :highlighted "Kunnskapsdepartementet", :value "Kunnskapsdepartementet"} 89 | {:count 392, :highlighted "Justis- og beredskapsdepartementet", :value "Justis- og beredskapsdepartementet"} 90 | {:count 322, :highlighted "Kommunal- og distriktsdepartementet", :value "Kommunal- og distriktsdepartementet"}], 91 | :field_name "orgs_long", 92 | :sampled false, 93 | :stats {:total_values 241}}]}]}) 94 | 95 | (def test-facet-results 96 | {:results 97 | [{:facet_counts 98 | [{:counts 99 | [{:count 337, :highlighted "Tildelingsbrev", :value "Tildelingsbrev"} 100 | {:count 111, :highlighted "Årsrapport", :value "Årsrapport"} 101 | {:count 92, :highlighted "Evaluering", :value "Evaluering"}], 102 | :field_name "type", 103 | :sampled false, 104 | :stats {:total_values 3}} 105 | {:counts 106 | [{:count 161, :value "Norges Forskningsråd"} 107 | {:count 394, :value "Kunnskapsdepartementet"}], 108 | :field_name "orgs_long", 109 | :sampled false, 110 | :stats {:total_values 241}}]}]}) 111 | 112 | (def test-entity {:id "71e1adbe-2116-478d-92b8-40b10a612d7b" 113 | :name "Kunnskapsassistent - DEV" 114 | :image "kudos-logo.png" 115 | :docs-collection "NEXT_kudos_docs" 116 | :chunks-collection "NEXT_kudos_chunks" 117 | :phrases-collection "NEXT_kudos_phrases" 118 | :phrase-gen-prompt "keyword-search" 119 | :reasoning-languages ["en" "no"] 120 | :prompt ""}) 121 | 122 | (rag/options (:results test-facet-results)) 123 | 124 | ;; Base case: nothing selected 125 | (= (->> (rag/fetch-facets test-entity 126 | {:type :typesense 127 | :fields [{:type :multiselect 128 | :expanded? true 129 | :selected-options #{} 130 | :field "type"} 131 | {:type :multiselect 132 | :expanded? true 133 | :selected-options #{} 134 | :field "orgs_long"}]}) 135 | :ui/fields 136 | (mapv (juxt :field :options)) 137 | set) 138 | #{["type" [{:count 32, :value "Tildelingsbrev", :selected? false} {:count 1, :value "Instruks", :selected? false} {:count 4, :value "Evaluering", :selected? false} {:count 10, :value "Årsrapport", :selected? false}]] 139 | 140 | ["orgs_short" [{:count 13, :value "Digdir", :selected? false} {:count 1, :value "NFD", :selected? false} {:count 32, :value "DFD", :selected? false} {:count 8, :value "DSB", :selected? false} {:count 1, :value "NGU", :selected? false} {:count 9, :value "KDD", :selected? false} {:count 8, :value "JD", :selected? false} {:count 1, :value "Statsforvalterens fellestjenester", :selected? false} {:count 1, :value "NFR", :selected? false} {:count 22, :value "Departementenes sikkerhets- og serviceorganisasjon", :selected? false}]]}) 141 | 142 | (->> (rag/fetch-facets test-entity 143 | {:type :typesense 144 | :fields [{:type :multiselect 145 | :expanded? true 146 | :selected-options #{"Tildelingsbrev"} 147 | :field "type"} 148 | {:type :multiselect 149 | :expanded? true 150 | :selected-options #{} 151 | :field "orgs_short"}]}) 152 | :ui/fields 153 | (mapv (juxt :field :options)) 154 | set)) 155 | 156 | (comment 157 | 158 | 159 | (def docs-collection-name "KUDOS_docs_2025-01-08") 160 | (def docs-collection-name "TEST_kudos_docs") 161 | 162 | (def input-filter-maps 163 | [;; n 0 164 | {:type :typesense 165 | :max-options 30 166 | :fields [{:type :multiselect 167 | :selected-options #{"Bufdir"} 168 | :field "orgs_short"} 169 | {:type :multiselect 170 | :selected-options #{} 171 | :field "type"}]} 172 | ;; n 1 173 | {:type :typesense 174 | :max-options 30 175 | :fields [{:type :multiselect 176 | :selected-options #{"Bufdir"} 177 | :field "orgs_short"} 178 | {:type :multiselect 179 | :selected-options #{"Årsrapport"} 180 | :field "type"}]} 181 | 182 | ;; n 2 183 | {:type :typesense 184 | :fields [{:type :multiselect 185 | :expanded? true 186 | :selected-options #{} 187 | :field "type"} 188 | {:type :multiselect 189 | :expanded? true 190 | :selected-options #{} 191 | :field "orgs_long"}]} 192 | 193 | ;; n 3 194 | {:type :typesense 195 | :fields [{:type :multiselect 196 | :expanded? true 197 | :selected-options #{} 198 | :field "type"} 199 | {:type :multiselect 200 | :expanded? true 201 | :selected-options #{"Kunnskapsdepartementet" "Norges Forskningsråd"} 202 | :field "orgs_long"}]} 203 | 204 | ;; n 4 205 | {:type :typesense 206 | :fields [{:type :multiselect 207 | :expanded? true 208 | :selected-options #{} 209 | :field "type"} 210 | {:type :multiselect 211 | :expanded? true 212 | :selected-options #{"Norges Forskningsråd"} 213 | :field "orgs_long"}]} 214 | ;; 215 | ]) 216 | 217 | ;; (def output-search-config 218 | ;; (filter-map->typesense-facet-multi-search 219 | ;; input-filter-map 220 | ;; "KUDOS_docs_2024-12-10")) 221 | 222 | 223 | (def target-search-config 224 | [;; n 0 225 | {:searches 226 | [{:q "*", 227 | :query_by "doc_num", 228 | :facet_by "orgs_short,type", 229 | :page 1, 230 | :filter_by "orgs_short:=[`Bufdir`]", 231 | :max_facet_values 10, 232 | :per_page 6, 233 | :collection "KUDOS_docs_2025-01-08"} 234 | {:query_by "doc_num", 235 | :collection "KUDOS_docs_2025-01-08", 236 | :q "*", 237 | :facet_by "orgs_short", 238 | :max_facet_values 10, 239 | :page 1}]} 240 | 241 | ;; n 1 242 | {:searches 243 | [{:q "*", 244 | :query_by "doc_num", 245 | :facet_by "orgs_short,type", 246 | :page 1, 247 | :filter_by "orgs_short: [`Bufdir`] && type:=[`Årsrapport`]", 248 | :max_facet_values 10, 249 | :per_page 6, 250 | :collection "KUDOS_docs_2025-01-08"} 251 | {:query_by "doc_num", 252 | :collection "KUDOS_docs_2025-01-08", 253 | :q "*", 254 | :facet_by "orgs_short", 255 | :filter_by "type:=[`Årsrapport`]", 256 | :max_facet_values 10, 257 | :page 1} 258 | {:query_by "doc_num", 259 | :collection "KUDOS_docs_2025-01-08", 260 | :q "*", 261 | :facet_by "type", 262 | :filter_by "orgs_short:=[`Bufdir`]", 263 | :max_facet_values 10, 264 | :page 1}]} 265 | 266 | ;; n 2 267 | {:searches 268 | [{:query_by "doc_num", 269 | :collection "KUDOS_docs_2025-01-08", 270 | :q "*", 271 | :facet_by "orgs_long,type", 272 | :max_facet_values 10, 273 | :page 1, 274 | :per_page 6}]} 275 | 276 | ;; n 3 277 | {:searches 278 | [{:q "*", 279 | :query_by "doc_num", 280 | :facet_by "orgs_long,type", 281 | :page 1, 282 | :filter_by 283 | "orgs_long:=[`Kunnskapsdepartementet`,`Norges Forskningsråd`]", 284 | :max_facet_values 10, 285 | :per_page 6, 286 | :collection "KUDOS_docs_2025-01-08"} 287 | {:query_by "doc_num", 288 | :collection "KUDOS_docs_2025-01-08", 289 | :q "*", 290 | :facet_by "orgs_long", 291 | :max_facet_values 10, 292 | :page 1}]} 293 | 294 | ;; n 4 295 | {:searches 296 | [{:q "*", 297 | :query_by "doc_num", 298 | :facet_by "orgs_long,type", 299 | :page 1, 300 | :filter_by 301 | "orgs_long:=[`Norges Forskningsråd`]", 302 | :max_facet_values 10, 303 | :per_page 6, 304 | :collection "KUDOS_docs_2025-01-08"} 305 | {:query_by "doc_num", 306 | :collection "KUDOS_docs_2025-01-08", 307 | :q "*", 308 | :facet_by "orgs_long", 309 | :max_facet_values 10, 310 | :page 1}]} 311 | 312 | ;; 313 | ]) 314 | 315 | (def n 4) 316 | 317 | (defn relevant-search-keys [{:keys [searches]}] 318 | (mapv 319 | (fn [{:keys [facet_by filter_by]}] 320 | (into {} 321 | (keep identity 322 | [(when (not-empty facet_by) 323 | [:facet_by facet_by]) 324 | (when (not-empty filter_by) 325 | [:filter_by filter_by])]))) 326 | searches)) 327 | 328 | (= 329 | ;; expected 330 | (relevant-search-keys (nth target-search-config n)) 331 | ;; actual 332 | (relevant-search-keys (filter-map->typesense-facet-multi-search 333 | (nth input-filter-maps n) 334 | docs-collection-name))) 335 | 336 | (print-diff 337 | 338 | ;; actual 339 | #_(relevant-search-keys) (filter-map->typesense-facet-multi-search 340 | (nth input-filter-maps n) 341 | docs-collection-name) 342 | ;; expected 343 | #_(relevant-search-keys) (nth target-search-config n)) 344 | 345 | 346 | (def conversation-entity {:id "71e1adbe-2116-478d-92b8-40b10a612d7b" 347 | :name "Kunnskapsassistent - test" 348 | :image "kudos-logo.png" 349 | :docs-collection "NEXT_kudos_docs" 350 | :chunks-collection "NEXT_kudos_chunks" 351 | :phrases-collection "NEXT_kudos_phrases" 352 | :phrase-gen-prompt "keyword-search" 353 | :reasoning-languages ["en" "no"] 354 | :prompt ""}) 355 | 356 | (def searches (filter-map->typesense-facet-multi-search 357 | (nth input-filter-maps n) 358 | (:docs-collection conversation-entity))) 359 | (def searches {:searches 360 | [{:q "tildelingsbrevet for 2022", 361 | :sort_by "_text_match:desc", 362 | :limit 20, 363 | :exclude_fields "phrase_vec", 364 | ;; :filter_by "$NEXT_kudos_docs(type:=[`Tildelingsbrev`] && orgs_long:=[`Barne-, ungdoms- og familiedirektoratet`])", 365 | 366 | ;; works 367 | ;; :filter_by "$NEXT_kudos_docs(type:=[`Tildelingsbrev`] )", 368 | 369 | ;; does NOT work 370 | :filter_by "$NEXT_kudos_docs(orgs_long:=[`Barne-, ungdoms- og familiedirektoratet`])", 371 | ;; :filter_by "$NEXT_kudos_docs(doc_num:=`26212`)", 372 | :prioritize_exact_match false, 373 | :drop_tokens_threshold 5, 374 | :include_fields "chunk_id,search_phrase", 375 | :collection "NEXT_kudos_phrases"}]}) 376 | (try 377 | (let [results (:results (ts-client/multi-search 378 | ts-cfg searches 379 | {:query_by "search_phrase,phrase_vec"})) 380 | opts (options results)] 381 | results) 382 | (catch Exception e 383 | (println "Error in fetch-facets:" 384 | (.getMessage e) "Error: " (str e) "queries:" searches))) 385 | 386 | (ts-client/multi-search ts-cfg 387 | {:searches [{:q "Instruks help", 388 | :sort_by "_text_match:desc", :limit 20, 389 | :exclude_fields "phrase_vec", 390 | :filter_by "$KUDOS_docs_2024-12-10(type:=['Instruks'])", 391 | :prioritize_exact_match false, :drop_tokens_threshold 5, 392 | :include_fields "chunk_id,search_phrase", 393 | :collection "KUDOS_phrases_2024-12-10"}]} 394 | {:query_by "chunk_id"}) 395 | 396 | (fetch-facets conversation-entity 397 | (filter-map->typesense-facet-multi-search 398 | (nth input-filter-maps n) 399 | docs-collection-name)) 400 | ) 401 | 402 | 403 | -------------------------------------------------------------------------------- /src/chat_app/server_jetty.clj: -------------------------------------------------------------------------------- 1 | (ns chat-app.server-jetty 2 | "Electric integrated into a sample ring + jetty app." 3 | (:require 4 | [chat-app.auth :as auth] 5 | [models.db :as db :refer [delayed-connection]] 6 | [hiccup2.core :as h] 7 | [ruuter.core :as ruuter] 8 | [datahike.core :as d] 9 | ;; Electric 10 | [clojure.edn :as edn] 11 | [clojure.java.io :as io] 12 | [clojure.string :as str] 13 | [clojure.tools.logging :as log] 14 | [contrib.assert :refer [check]] 15 | [hyperfiddle.electric-ring-adapter :as electric-ring] 16 | [ring.adapter.jetty :as ring] 17 | [ring.middleware.content-type :refer [wrap-content-type]] 18 | [ring.middleware.cookies :as cookies] 19 | [ring.middleware.params :refer [wrap-params]] 20 | [ring.middleware.resource :refer [wrap-resource]] 21 | [ring.util.response :as res] 22 | [chat-app.webauthn :as webauthn]) 23 | (:import 24 | (org.eclipse.jetty.server.handler.gzip GzipHandler) 25 | (org.eclipse.jetty.websocket.server.config JettyWebSocketServletContainerInitializer JettyWebSocketServletContainerInitializer$Configurator))) 26 | 27 | 28 | ;; TODO: add the actual expiry we want on the cookie 29 | (defn set-http-only-cookie 30 | "Helper function to set an HTTP-only cookie with the JWT token." 31 | [response jwt-token] 32 | (assoc-in response [:cookies "auth-token"] 33 | {:value jwt-token 34 | :path "/" 35 | :http-only true 36 | :same-site :strict 37 | :max-age 200000})) 38 | 39 | (defn remove-http-only-cookie [response] 40 | (assoc-in response [:cookies "auth-token"] 41 | {:value "" 42 | :path "/" 43 | :http-only true 44 | :same-site :strict 45 | :max-age 0})) ;; Set max-age to 0 to remove the cookie 46 | 47 | 48 | (defn confirm-email-page [email] 49 | (res/response 50 | (str (h/html 51 | [:html 52 | [:head 53 | [:meta {:charset "utf-8"}] 54 | [:title "Sjekk din innboks"] 55 | [:link {:rel "stylesheet" 56 | :href "/styles.css"}] 57 | [:link {:rel "icon" 58 | :type "image/svg+xml" 59 | :href "digdir_icon.svg"}]] 60 | [:body {:class "flex justify-center items-center bg-slate-100"} 61 | [:div {:class "flex flex-col gap-4"} 62 | [:h1 {:class "text-4xl font-bold text-center"} "Sjekk innboksen din"] 63 | [:p "Vi har sendt din påloggingskode til: " [:b email] "."] 64 | [:p "Koden er gyldig i et begrenset tidsperiode."] 65 | [:form {:action "/auth/confirm-email" 66 | :method "post" 67 | :class "flex flex-col gap-4"} 68 | [:input {:type "hidden" :name "email" :value email}] 69 | 70 | [:input {:type "text" :name "confirmation-code" :placeholder "6-sifret kode" 71 | :required true :maxlength "6" :pattern "\\d{6}" :title "Skriv din 6-sifret kode" 72 | :autofocus true}] 73 | [:button {:type "submit" 74 | :class "px-4 py-2 bg-black hover:bg-slate-800 text-white rounded"} 75 | "Logg på"]] 76 | #_[:p "Codes: " (str @auth/confirmation-codes)]]]])))) 77 | 78 | (defn auth-model [{:keys [title action]}] 79 | (str (h/html 80 | [:html 81 | [:head 82 | [:title title] 83 | [:link {:rel "stylesheet" 84 | :href "/styles.css"}] 85 | [:link {:rel "icon" 86 | :type "image/svg+xml" 87 | :href "digdir_icon.svg"}]] 88 | [:body {:class "flex justify-center items-center bg-slate-100"} 89 | [:div {:class "p-8 shadow bg-white flex flex-col gap-4 rounded-lg w-96"} 90 | [:h1 {:class "text-2xl font-bold text-center"} title] 91 | [:form {:action "/auth" 92 | :method "post" 93 | :class "flex flex-col gap-4"} 94 | 95 | [:input {:type "email" :id "email" :name "email" :placeholder "E-post adresse" :required true :autofocus true}] 96 | [:button {:type "submit" 97 | :class "px-4 py-2 bg-black hover:bg-slate-800 text-white rounded"} 98 | "Fortsett"]] 99 | (if (= action "/auth") 100 | [:p "Har du en konto allerede? " [:a {:href "/login" 101 | :class "text-green-500"} "Sign in"]] 102 | [:p "Trenger du en brukerkonto? " [:a {:href "/auth" 103 | :class "text-green-500"} "Sign up"]])]]]))) 104 | 105 | 106 | (defn email-signup [] 107 | (res/response 108 | (auth-model {:title "Opprett en konto" 109 | :action "/auth"}))) 110 | 111 | (defn login [] 112 | (res/response 113 | (auth-model {:title "Velkommen tilbake" 114 | :action "/login"}))) 115 | 116 | (def routes 117 | [{:path "/auth" 118 | :method :get 119 | :response (email-signup)} 120 | 121 | {:path "/auth" 122 | :method :post 123 | :response (fn [ring-req] 124 | (let [conn @delayed-connection 125 | email (get-in ring-req [:params "email"])] 126 | (if email 127 | (if (auth/approved-domain? email) 128 | (if-let [passkey (webauthn/get-user-key @conn email)] 129 | (do 130 | (println "passkey: " passkey) 131 | (let [cose-map (edn/read-string passkey)] 132 | (println "this is the cose map" 133 | (-> cose-map 134 | (update-in [:public-key -2] webauthn/base64-to-byte-array) 135 | (update-in [:public-key -3] webauthn/base64-to-byte-array))) 136 | (when-let [credentials (:credential-id cose-map)] 137 | ;; TODO: use something more secure to isolating the session request 138 | (let [session-id email 139 | key-request (webauthn/map-to-base64 140 | (webauthn/create-public-key-request-options session-id email 141 | {:rp-id "localhost" 142 | :allowed-credentials [credentials]}))] 143 | (println "This is request options as: " key-request) 144 | (res/redirect (str "/auth/passkey?email=" email "&request=" key-request)))))) 145 | (do 146 | (println "this is the email passed to the backend: " email) 147 | (auth/create-new-user {:email email}) 148 | 149 | ;; either: just generate a code 150 | ;; (auth/generate-confirmation-code email) 151 | 152 | ;; or: generate a code and sent it by email 153 | ;; TODO: use a static sender email, rather than a matching email (required during test mode) 154 | (auth/send-confirmation-code "kunnskap@digdir.cloud" email (auth/generate-confirmation-code email)) 155 | 156 | (res/redirect (str "/auth/confirm-email?email=" email)))) 157 | (res/redirect "/not-approved")) 158 | (res/status (res/response "

Error

Email address is required.

") 400))))} 159 | 160 | {:path "/not-approved" 161 | :method :get 162 | :response (res/response (str (h/html 163 | [:html 164 | [:body 165 | [:h1 "Påloggingsfeil"] 166 | [:p "Beklager, den oppgitte epost adresse har ikke tilgang."]]])))} 167 | 168 | {:path "/login" 169 | :method :get 170 | :response (login)} 171 | 172 | {:path "/logout" 173 | :method :get 174 | :response (let [response (res/redirect "/")] 175 | (remove-http-only-cookie response))} 176 | 177 | {:path "/auth/passkey" 178 | :method :get 179 | :response (res/response (str (h/html 180 | [:html 181 | [:body 182 | [:h1 "Passkey"] 183 | [:button {:id "get-passkey" 184 | :class "px-4 py-2 bg-black hover:bg-slate-800 text-white rounded"} 185 | "Login with passkey"] 186 | #_[:script {:src "/js/webauthn.js"}]]])))} 187 | 188 | {:path "/auth/confirm-email" 189 | :method :get 190 | :response (fn [ring-req] 191 | (let [email (get-in ring-req [:query-params "email"])] 192 | (if email 193 | (confirm-email-page email) 194 | (res/status (res/response "

Error

Email address is missing.

") 400))))} 195 | 196 | {:path "/auth/confirm-email" 197 | :method :post 198 | :response (fn [ring-req] 199 | (let [{:strs [email confirmation-code]} (:params ring-req)] 200 | (if (and email confirmation-code) 201 | (if (auth/valid-code? email confirmation-code) 202 | (let [jwt-token (auth/create-token email (auth/create-expiry {:multiplier 48 203 | :timespan :hours})) 204 | response (res/redirect "/")] 205 | (swap! auth/confirmation-codes dissoc email) 206 | (set-http-only-cookie response jwt-token)) 207 | (res/response (str (h/html 208 | [:html 209 | [:head 210 | [:meta {:charset "utf-8"}]] 211 | [:body 212 | [:h1 "Error"] 213 | [:p "Koden er ugyldig eller utgått."]]])))) 214 | (res/status (res/response "

Error

Mangler epost eller bekreftelseskoden.

") 400))))}]) 215 | 216 | 217 | 218 | (defn wrap-auth 219 | "A basic path-based routing middleware" 220 | [next-handler] 221 | (fn [ring-req] 222 | (let [auth-token (get-in ring-req [:cookies "auth-token" :value]) 223 | valid-token? (:valid (auth/verify-token auth-token)) 224 | uri (:uri ring-req) 225 | auth-route? (contains? (set (map :path routes)) uri) 226 | static-url? (re-find #"\.(css|js|png|jpg|jpeg|gif|ico|svg)$" uri)] 227 | (cond 228 | (= uri "/logout") (let [response (res/redirect "/")] 229 | (remove-http-only-cookie response)) 230 | valid-token? (next-handler ring-req) 231 | static-url? (next-handler ring-req) 232 | auth-route? (ruuter/route routes ring-req) 233 | (not valid-token?) (res/redirect "/login") 234 | :else (res/not-found "Not found"))))) 235 | 236 | 237 | 238 | ;;; Electric integration 239 | 240 | (defn electric-websocket-middleware 241 | "Open a websocket and boot an Electric server program defined by `entrypoint`. 242 | Takes: 243 | - a ring handler `next-handler` to call if the request is not a websocket upgrade (e.g. the next middleware in the chain), 244 | - a `config` map eventually containing {:hyperfiddle.electric/user-version } to ensure client and server share the same version, 245 | - see `hyperfiddle.electric-ring-adapter/wrap-reject-stale-client` 246 | - an Electric `entrypoint`: a function (fn [ring-request] (e/boot-server {} my-ns/My-e-defn ring-request)) 247 | " 248 | [next-handler config entrypoint] 249 | ;; Applied bottom-up 250 | (-> (electric-ring/wrap-electric-websocket next-handler entrypoint) ; 5. connect electric client 251 | ; 4. this is where you would add authentication middleware (after cookie parsing, before Electric starts) 252 | (cookies/wrap-cookies) ; 3. makes cookies available to Electric app 253 | (electric-ring/wrap-reject-stale-client config) ; 2. reject stale electric client 254 | (wrap-params))) ; 1. parse query params 255 | 256 | (defn get-modules [manifest-path] 257 | (when-let [manifest (io/resource manifest-path)] 258 | (let [manifest-folder (when-let [folder-name (second (rseq (str/split manifest-path #"\/")))] 259 | (str "/" folder-name "/"))] 260 | (->> (slurp manifest) 261 | (edn/read-string) 262 | (reduce (fn [r module] (assoc r (keyword "hyperfiddle.client.module" (name (:name module))) 263 | (str manifest-folder (:output-name module)))) {}))))) 264 | 265 | (defn template 266 | "In string template `
$:foo/bar$
`, replace all instances of $key$ 267 | with target specified by map `m`. Target values are coerced to string with `str`. 268 | E.g. (template \"
$:foo$
\" {:foo 1}) => \"
1
\" - 1 is coerced to string." 269 | [t m] (reduce-kv (fn [acc k v] (str/replace acc (str "$" k "$") (str v))) t m)) 270 | 271 | ;;; Template and serve index.html 272 | 273 | (defn wrap-index-page 274 | "Server the `index.html` file with injected javascript modules from `manifest.edn`. 275 | `manifest.edn` is generated by the client build and contains javascript modules 276 | information." 277 | [next-handler config] 278 | (fn [ring-req] 279 | (if-let [response (res/resource-response (str (check string? (:resources-path config)) "/index.html"))] 280 | (if-let [bag (merge config (get-modules (check string? (:manifest-path config))))] 281 | (-> (res/response (template (slurp (:body response)) bag)) ; TODO cache in prod mode 282 | (res/content-type "text/html") ; ensure `index.html` is not cached 283 | (res/header "Cache-Control" "no-store") 284 | (res/header "Last-Modified" (get-in response [:headers "Last-Modified"]))) 285 | (-> (res/not-found (pr-str ::missing-shadow-build-manifest)) ; can't inject js modules 286 | (res/content-type "text/plain"))) 287 | ;; index.html file not found on classpath 288 | (next-handler ring-req)))) 289 | 290 | (defn not-found-handler [_ring-request] 291 | (-> (res/not-found "Not found") 292 | (res/content-type "text/plain"))) 293 | 294 | (defn http-middleware [config] 295 | ;; these compose as functions, so are applied bottom up 296 | (-> not-found-handler 297 | (wrap-index-page config) ; 3. otherwise fallback to default page file 298 | (wrap-resource (:resources-path config)) ; 2. serve static file from classpath 299 | (wrap-content-type) ; 1. detect content (e.g. for index.html) 300 | (wrap-auth) 301 | )) 302 | 303 | (defn middleware [config entrypoint] 304 | (-> (http-middleware config) ; 2. otherwise, serve regular http content 305 | (electric-websocket-middleware config entrypoint))) ; 1. intercept websocket upgrades and maybe start Electric 306 | 307 | (defn- add-gzip-handler! 308 | "Makes Jetty server compress responses. Optional but recommended." 309 | [server] 310 | (.setHandler server 311 | (doto (GzipHandler.) 312 | #_(.setIncludedMimeTypes (into-array ["text/css" "text/plain" "text/javascript" "application/javascript" "application/json" "image/svg+xml"])) ; only compress these 313 | (.setMinGzipSize 1024) 314 | (.setHandler (.getHandler server))))) 315 | 316 | (defn- configure-websocket! 317 | "Tune Jetty Websocket config for Electric compat." [server] 318 | (JettyWebSocketServletContainerInitializer/configure 319 | (.getHandler server) 320 | (reify JettyWebSocketServletContainerInitializer$Configurator 321 | (accept [_this _servletContext wsContainer] 322 | (.setIdleTimeout wsContainer (java.time.Duration/ofSeconds 60)) 323 | (.setMaxBinaryMessageSize wsContainer (* 100 1024 1024)) ; 100M - temporary 324 | (.setMaxTextMessageSize wsContainer (* 100 1024 1024)) ; 100M - temporary 325 | )))) 326 | 327 | (defn start-server! [entrypoint 328 | {:keys [port host] 329 | :or {port 8080, host "0.0.0.0"} 330 | :as config}] 331 | (let [server (ring/run-jetty (middleware config entrypoint) 332 | (merge {:port port 333 | :join? false 334 | :configurator (fn [server] 335 | (configure-websocket! server) 336 | (add-gzip-handler! server))} 337 | config))] 338 | (log/info "👉" (str "http://" host ":" (-> server (.getConnectors) first (.getPort)))) 339 | server)) 340 | 341 | -------------------------------------------------------------------------------- /src/chat_app/webauthn.cljc: -------------------------------------------------------------------------------- 1 | (ns chat-app.webauthn 2 | #?(:clj (:import java.util.Base64 3 | [java.nio ByteBuffer] 4 | [java.security Signature KeyPairGenerator KeyFactory] 5 | [java.nio.charset StandardCharsets] 6 | [java.security.spec ECPoint ECPublicKeySpec ECGenParameterSpec] 7 | [java.security.interfaces ECPublicKey])) 8 | (:require #?(:clj [clojure.data.json :as json]) 9 | #?(:clj [clj-cbor.core :as cbor]) 10 | #?(:clj [pandect.algo.sha256 :as sha]) 11 | #?(:clj [datahike.core :as d]) 12 | #?(:cljs [goog.dom :as gdom]) 13 | [nano-id.core :refer [nano-id]])) 14 | 15 | (def challenge-settings {:timeout 60000 16 | :prune-interval 10000}) 17 | 18 | #?(:clj (defonce !create-opts (atom nil))) 19 | #?(:clj (defonce !session-challenges (atom {}))) 20 | #?(:clj (defonce !registered-users (atom {}))) 21 | 22 | #?(:cljs (defonce !created-key (atom nil))) 23 | ;; Security focused libraries like Duo issues alerts with repeatadly failed challenges 24 | ;; {:user-session {:created-at "timestamp of when challenge was issued" 25 | ;; :challenge "the challenge id"}} 26 | 27 | #?(:cljs (defn str->Uint8Array [s] 28 | (js/Uint8Array. (clj->js (map #(.charCodeAt % 0) (seq s)))))) 29 | 30 | #?(:cljs (defn Uint8Array->str [arr] 31 | (apply str (map #(js/String.fromCharCode %) arr)))) 32 | 33 | #?(:cljs (defn str->base64 [s] 34 | (js/btoa s))) 35 | 36 | #?(:cljs (defn base64->str [base64] 37 | (js/atob base64))) 38 | 39 | #?(:cljs (defn prepare-for-creation [auth-options] 40 | (clj->js (-> auth-options 41 | (update-in [:challenge] str->Uint8Array) 42 | (update-in [:user :id] str->Uint8Array))))) 43 | 44 | #?(:cljs (defn prepare-for-login [auth-options] 45 | (println "This is the challenge before prepped" (:challenge auth-options)) 46 | (clj->js (-> auth-options 47 | (update-in [:allowCredentials 0 :id] #(js/Uint8Array. %)) 48 | (update-in [:challenge] str->Uint8Array))))) 49 | 50 | #?(:cljs (defn array-buffer->json [^js/ArrayBuffer buffer] 51 | (let [decoder (js/TextDecoder. "utf-8") 52 | json-str (.decode decoder (js/Uint8Array. buffer))] 53 | (js/JSON.parse json-str)))) 54 | 55 | #?(:cljs (defn array-buffer->base64 [^js/ArrayBuffer buffer] 56 | (let [uint8-array (js/Uint8Array. buffer)] 57 | (js/btoa (apply str (map #(js/String.fromCharCode %) uint8-array)))))) 58 | 59 | #?(:cljs (defn serialize-public-key-credential [^js pkc] 60 | (let [raw-id (array-buffer->base64 (.-rawId pkc)) 61 | client-data-json (array-buffer->base64 (.-clientDataJSON (.-response pkc))) 62 | attestation-object (array-buffer->base64 (.-attestationObject (.-response pkc)))] 63 | {:id (.-id pkc) 64 | :rawId raw-id 65 | :type (.-type pkc) 66 | :authenticatorAttachment (.-authenticatorAttachment pkc) 67 | :response {:clientDataJSON client-data-json 68 | :attestationObject attestation-object}}))) 69 | 70 | 71 | #?(:cljs (defn serialize-login-key-credential [^js pkc] 72 | (let [auth-assert-resp (.-response pkc) 73 | client-data-json (array-buffer->base64 (.-clientDataJSON auth-assert-resp)) 74 | signature (array-buffer->base64 (.-signature auth-assert-resp)) 75 | user-handle (array-buffer->base64 (.-userHandle auth-assert-resp)) 76 | authenticator-data (array-buffer->base64 (.-authenticatorData auth-assert-resp)) 77 | raw-id (array-buffer->base64 (.-rawId pkc))] 78 | {:id (.-id pkc) 79 | :rawId raw-id 80 | :type (.-type pkc) 81 | :response {:clientDataJSON client-data-json 82 | :signature signature 83 | :user-handle user-handle 84 | :authenticator-data authenticator-data}}))) 85 | 86 | #?(:cljs (defn create-credential [opts cb] 87 | (-> (.create js/navigator.credentials (clj->js {:publicKey opts})) 88 | (.then (fn [credential] 89 | (cb credential))) 90 | (.catch (fn [error] 91 | (js/console.error "Error creating credential:" error)))))) 92 | 93 | #?(:cljs (defn get-credential [opts cb] 94 | (-> (.get js/navigator.credentials (clj->js {:publicKey opts})) 95 | (.then (fn [credential] 96 | (cb credential))) 97 | (.catch (fn [error] 98 | (js/console.error "Error creating credential:" error)))))) 99 | 100 | ;; Backend 101 | 102 | #?(:clj (defn prune-expired [state-atom timeout] 103 | (let [now (System/currentTimeMillis)] 104 | (swap! state-atom 105 | #(into {} (filter (fn [[_ {:keys [created-at]}]] 106 | (>= created-at (- now timeout))) %)))))) 107 | 108 | #?(:clj (defn start-pruning-loop [] 109 | (println "Starting prune loop") 110 | (future 111 | (while true 112 | (Thread/sleep (:prune-interval challenge-settings)) 113 | ;; (println "pruning challenges") 114 | (prune-expired !session-challenges (:timeout challenge-settings)))))) 115 | 116 | #?(:clj (start-pruning-loop)) 117 | 118 | #?(:clj (defn bytes-to-uint16 [byte-seq] 119 | (let [buffer (java.nio.ByteBuffer/wrap (byte-array byte-seq))] 120 | (.getShort buffer 0)))) 121 | 122 | #?(:clj (defn parse-auth-data [auth-data] 123 | (let [id-len-bytes (subvec (vec auth-data) 53 55) 124 | credential-id-length (bytes-to-uint16 id-len-bytes) 125 | credential-id (subvec (vec auth-data) 55 (+ 55 credential-id-length)) 126 | public-key-bytes (subvec (vec auth-data) (+ 55 credential-id-length)) 127 | public-key-object (cbor/decode (byte-array public-key-bytes))] 128 | {:credential-id credential-id 129 | :public-key public-key-object}))) 130 | 131 | #?(:clj (defn parse-credential [data] 132 | (let [base64->bytes #(.decode (Base64/getDecoder) %) 133 | base64->str #(String. (base64->bytes %) "UTF-8") 134 | decode-attestation-object #(clojure.walk/keywordize-keys (cbor/decode (base64->bytes %))) 135 | parsed-data (-> data 136 | (update-in [:response :clientDataJSON] #(json/read-str (base64->str %) :key-fn keyword)) 137 | (update-in [:response :clientDataJSON :challenge] base64->str) 138 | (update-in [:response :attestationObject] decode-attestation-object) 139 | (update-in [:response :attestationObject :authData] parse-auth-data))] 140 | parsed-data))) 141 | 142 | #?(:clj (defn parse-credential-login [data] 143 | (let [base64->bytes #(.decode (Base64/getDecoder) %) 144 | bytes->str #(String. % "UTF-8") 145 | base64->str #(String. (base64->bytes %) "UTF-8") 146 | resp-bytes (update-vals (:response data) base64->bytes) 147 | client-data-hash (sha/sha256-bytes (:clientDataJSON resp-bytes)) 148 | authenticator-data (:authenticator-data resp-bytes) 149 | concat-bytes (byte-array (concat authenticator-data client-data-hash)) 150 | parsed-data (-> data 151 | (assoc :response resp-bytes) 152 | (update-in [:response :clientDataJSON] #(json/read-str (bytes->str %) :key-fn keyword)) 153 | (update-in [:response :clientDataJSON :challenge] base64->str) 154 | (update-in [:response :user-handle] bytes->str) 155 | (assoc :concat-bytes concat-bytes))] 156 | parsed-data))) 157 | 158 | #?(:clj (defn verify-challenge [session-id credential-data] 159 | (let [session-challenge (get-in @!session-challenges [session-id :challenge]) 160 | resp-challenge (get-in credential-data [:response :clientDataJSON :challenge])] 161 | (cond 162 | (= resp-challenge session-challenge) {:status :success} 163 | (nil? session-challenge) {:status :error 164 | :message "No challenge found in session" 165 | :session-id session-id} 166 | (not= resp-challenge session-challenge) {:status :error 167 | :message "Challenge is different to session challenge" 168 | :expected session-challenge 169 | :received resp-challenge})))) 170 | 171 | #?(:clj 172 | (defn create-public-key-options 173 | "Generates a WebAuthn `publicKeyCredentialCreationOptions` map used for registering a new credential. 174 | The map includes a challenge, relying party (RP) information, user information, and cryptographic options. 175 | Optionally supports user verification, authenticator type selection, attestation level, and timeout settings. 176 | 177 | Arguments: 178 | - `session-id`: Unique identifier for the user's session. 179 | - `options`: Map containing: 180 | - `:name` (string): The user's account name. 181 | - `:display-name` (string): A human-readable display name for the user. 182 | - `:rp-name` (string): Name of the relying party (RP, e.g., your website). 183 | - `:rp-id` (string): Relying party identifier (typically your domain). 184 | - `:authenticatorSelection` (map): Map specifying authenticator options such as: 185 | - `:authenticatorAttachment` (string): Choose between 'platform' or 'cross-platform'. 186 | - `:userVerification` (string): Level of user verification required ('required', 'preferred', or 'discouraged'). 187 | - `:attestation` (optional string): Level of attestation ('none', 'indirect', or 'direct'). Default is 'none'. 188 | - `:timeout` (optional number): Time (in ms) for the user to respond to the credential creation request." 189 | 190 | [session-id {:keys [name display-name rp-name rp-id authenticatorSelection attestation timeout]}] 191 | (let [challenge (nano-id) 192 | create-opts {:challenge challenge 193 | :rp {:name rp-name 194 | :id rp-id} 195 | :user {:id (nano-id) 196 | :name name 197 | :displayName display-name} 198 | :pubKeyCredParams [{:alg -7 :type "public-key"} ;Most common and considered most efficient and secure 199 | {:alg -257 :type "public-key"}] ;RS256 is necessary for compatibility with Microsoft Windows platform 200 | :authenticatorSelection authenticatorSelection 201 | :timeout (or timeout (:timeout challenge-settings)) 202 | ;; :attestation (or attestation "none") ; or directnot confirmed to work 203 | }] 204 | ;; Store the challenge for session validation 205 | (reset! !create-opts create-opts) 206 | (swap! !session-challenges assoc session-id {:created-at (System/currentTimeMillis) 207 | :challenge challenge 208 | :type :register 209 | :name name}) 210 | create-opts))) 211 | 212 | ;; Extra helper. People could do this themselves 213 | #?(:clj (defn handle-new-user-creds 214 | "Takes a callback to handle the auth data when challenge is verified. 215 | When the challenge is verified calls the callback and returns {:success true} 216 | When the challenge fails returns a map with the {:status :error} and helpful information, see verify-challenge." 217 | [session-id creds cb] 218 | (let [parsed-credential-obj (parse-credential creds) 219 | auth-data (get-in parsed-credential-obj [:response :attestationObject :authData]) 220 | username (get-in @!session-challenges [session-id :name]) 221 | result (verify-challenge session-id parsed-credential-obj)] 222 | (case (:status result) 223 | :success (do 224 | (swap! !session-challenges dissoc session-id) 225 | (cb username auth-data) 226 | {:success true}) 227 | :error result)))) 228 | 229 | #?(:clj (defn create-public-key-request-options [session-id name {:keys [challenge rp-id allowed-credentials]}] 230 | (let [challenge (nano-id)] 231 | (swap! !session-challenges assoc session-id {:created-at (System/currentTimeMillis) 232 | :challenge challenge 233 | :type :login 234 | :name name}) 235 | {:challenge challenge 236 | ;; :rpId rp-id ;; Same relying party ID used during registration 237 | :allowCredentials (mapv (fn [cred-id] 238 | {:type "public-key" 239 | :id cred-id}) 240 | allowed-credentials) ;; List of previously registered credential IDs 241 | :timeout 60000 ;; Optional timeout setting (e.g., 60 seconds) 242 | ;; :userVerification "preferred" 243 | }))) ;; Optional user verification requirement 244 | 245 | #?(:clj (defn verify-signature-bytes [public-key message-bytes signature-bytes] 246 | (let [signature-instance (Signature/getInstance "SHA256withECDSA")] 247 | (.initVerify signature-instance public-key) 248 | (.update signature-instance message-bytes) 249 | (.verify signature-instance signature-bytes)))) 250 | 251 | #?(:clj (defn cose->ec-public-key [cose-map] 252 | (let [[x y] [(BigInteger. 1 (byte-array (get cose-map -2))) 253 | (BigInteger. 1 (byte-array (get cose-map -3)))] 254 | point (ECPoint. x y) ;; Create ECPoint (x, y) 255 | 256 | ;; Initialize the key pair generator for EC curve P-256 (secp256r1) 257 | kpg (KeyPairGenerator/getInstance "EC") 258 | _ (.initialize kpg (ECGenParameterSpec. "secp256r1")) 259 | temp-key-pair (.generateKeyPair kpg) 260 | temp-public-key (.getPublic temp-key-pair) 261 | param-spec (.getParams ^ECPublicKey temp-public-key) 262 | key-factory (KeyFactory/getInstance "EC") 263 | key-spec (ECPublicKeySpec. point param-spec)] 264 | 265 | ;; Generate key 266 | (.generatePublic key-factory key-spec)))) 267 | 268 | #?(:clj (defn verify-credentials [session-id cose-map credentials] 269 | (let [creds (parse-credential-login credentials) 270 | result (verify-challenge session-id creds)] 271 | (case (:status result) 272 | :success (do 273 | (swap! !session-challenges dissoc session-id) 274 | (let [public-key (cose->ec-public-key cose-map) 275 | concat-bytes (:concat-bytes creds) 276 | sig (get-in creds [:response :signature])] 277 | (verify-signature-bytes public-key concat-bytes sig))) 278 | :error result)))) 279 | 280 | ;; New additions 281 | 282 | #?(:clj (defn byte-array-to-base64 [byte-array] 283 | (let [encoder (Base64/getEncoder)] 284 | (.encodeToString encoder byte-array)))) 285 | 286 | #?(:clj (defn base64-to-byte-array [base64-str] 287 | (let [decoder (Base64/getDecoder)] 288 | (.decode decoder base64-str)))) 289 | 290 | #?(:clj (defn get-user-key [db email] 291 | (d/q '[:find ?key . 292 | :in $ ?email 293 | :where 294 | [?e :user/email ?email] 295 | [?e :user/key ?key]] 296 | db email))) 297 | 298 | #?(:clj (defn map-to-base64 [m] 299 | (let [json-str (json/write-str m) 300 | json-bytes (.getBytes json-str "UTF-8")] 301 | (byte-array-to-base64 json-bytes)))) 302 | 303 | #?(:cljs (defn get-query-params [] 304 | (let [url-params (js/URLSearchParams. (.-search js/window.location))] 305 | (into {} (for [k (js->clj (.keys url-params))] 306 | [k (.get url-params k)]))))) 307 | 308 | #?(:cljs (defn handle-passkey-click [] 309 | (let [req-key (get (get-query-params) "request") 310 | parsed-req-key (js->clj (js/JSON.parse (base64->str req-key)) 311 | :keywordize-keys true) 312 | prepared-request (prepare-for-login parsed-req-key)] 313 | (js/console.log "Passkey button clicked!") 314 | (println req-key) 315 | (println (base64->str req-key)) 316 | (println "hey: " (keys parsed-req-key)) 317 | (println "parsed req key: " parsed-req-key) 318 | (println "prepped: " prepared-request) 319 | (get-credential prepared-request #(println "Hello key:" %)) 320 | #_(prepare-for-login (base64->str req-key))) 321 | ;; Add your passkey-related logic here 322 | )) 323 | 324 | #?(:cljs (defn init [] 325 | (let [button (gdom/getElement "get-passkey")] 326 | (when button 327 | (.addEventListener button "click" handle-passkey-click))))) 328 | 329 | ;; Ensure the function runs when the script is loaded 330 | #?(:cljs (init)) 331 | 332 | #?(:cljs (println "This is the webauthn code loaded")) 333 | 334 | 335 | 336 | -------------------------------------------------------------------------------- /src/models/db.cljc: -------------------------------------------------------------------------------- 1 | (ns models.db 2 | (:require [nano-id.core :refer [nano-id]] 3 | [chat-app.kit :as kit] 4 | [clojure.edn :as edn] 5 | [hyperfiddle.electric :as e] 6 | #?(:clj [datahike.api :as d]) 7 | #?(:clj [datahike-jdbc.core]) 8 | #?(:clj [aero.core :as aero]))) 9 | 10 | ;; TODO: 11 | ;; Remove the delayed connection and replace with environment variables for db creation 12 | 13 | #?(:clj 14 | (def config-filename (System/getenv "ENTITY_CONFIG_FILE"))) 15 | 16 | 17 | #?(:clj 18 | (defn load-config [] 19 | (let [_ (println (str "reading config from file: " config-filename))] 20 | (if config-filename 21 | (aero/read-config config-filename) 22 | (println (str "ENTITY_CONFIG_FILE env variable empty, unable to load config. This is expected during builds.")))))) 23 | 24 | #?(:clj 25 | (def !config (atom (load-config)))) 26 | 27 | #?(:clj 28 | (defn config [] @!config)) 29 | 30 | 31 | #?(:clj 32 | (def cfg (get-in (config) [:db (:db-env (config))]))) 33 | 34 | #?(:clj 35 | (defn update-config [new-entity-config] 36 | (let [current-config (config) 37 | current-entities (get-in current-config [:chat :entities]) 38 | updated-entities (map #(if (= (:id %) (:id new-entity-config)) 39 | (merge % new-entity-config) 40 | %) 41 | current-entities) 42 | updated-config (assoc-in current-config [:chat :entities] updated-entities)] 43 | (when config-filename 44 | (println "Updating config file...") 45 | (spit config-filename (pr-str updated-config)) 46 | (reset! !config updated-config) 47 | (println "Config updated successfully.")) 48 | updated-config))) 49 | 50 | (def dh-schema 51 | [;; Folder 52 | {:db/ident :folder/id 53 | :db/valueType :db.type/string 54 | :db/cardinality :db.cardinality/one 55 | :db/unique :db.unique/identity} 56 | {:db/ident :folder/name 57 | :db/valueType :db.type/string 58 | :db/cardinality :db.cardinality/one} 59 | 60 | ;; prompt.folder 61 | {:db/ident :prompt.folder/id 62 | :db/valueType :db.type/string 63 | :db/cardinality :db.cardinality/one 64 | :db/unique :db.unique/identity} 65 | {:db/ident :prompt.folder/name 66 | :db/valueType :db.type/string 67 | :db/cardinality :db.cardinality/one} 68 | 69 | ;; Prompt 70 | {:db/ident :prompt/id 71 | :db/valueType :db.type/string 72 | :db/cardinality :db.cardinality/one 73 | :db/unique :db.unique/identity} 74 | 75 | ;; Conversation 76 | {:db/ident :conversation/id 77 | :db/valueType :db.type/string 78 | :db/cardinality :db.cardinality/one 79 | :db/unique :db.unique/identity} 80 | {:db/ident :conversation/topic 81 | :db/valueType :db.type/string 82 | :db/cardinality :db.cardinality/one} 83 | {:db/ident :conversation/messages 84 | :db/valueType :db.type/ref 85 | :db/cardinality :db.cardinality/many} 86 | 87 | ;; Text message 88 | {:db/ident :message/id 89 | :db/valueType :db.type/string 90 | :db/cardinality :db.cardinality/one 91 | :db/unique :db.unique/identity} 92 | {:db/ident :message/text 93 | :db/valueType :db.type/string 94 | :db/cardinality :db.cardinality/one} 95 | {:db/ident :message/completion 96 | :db/valueType :db.type/boolean 97 | :db/cardinality :db.cardinality/one} 98 | 99 | ;; Filter message 100 | {:db/ident :message/id 101 | :db/valueType :db.type/string 102 | :db/cardinality :db.cardinality/one 103 | :db/unique :db.unique/identity} 104 | {:db/ident :message.filter/value 105 | :db/valueType :db.type/string ;; edn - clojure.edn/read-string 106 | :db/cardinality :db.cardinality/one} 107 | 108 | {:db/ident :active-key-name 109 | :db/valueType :db.type/string 110 | :db/cardinality :db.cardinality/one} 111 | {:db/ident :key/value 112 | :db/valueType :db.type/string 113 | :db/cardinality :db.cardinality/one} 114 | {:db/ident :user/id 115 | :db/valueType :db.type/string 116 | :db/unique :db.unique/identity 117 | :db/cardinality :db.cardinality/one} 118 | {:db/ident :user/email 119 | :db/valueType :db.type/string 120 | :db/unique :db.unique/identity 121 | :db/cardinality :db.cardinality/one}]) 122 | 123 | 124 | #?(:clj 125 | (defn init-db [] 126 | (if cfg 127 | (do 128 | (when-not (d/database-exists? cfg) 129 | (d/create-database cfg) 130 | (let [conn (d/connect cfg)] 131 | (d/transact conn {:tx-data dh-schema}))) 132 | (d/connect cfg)) 133 | (println (str "no db config loaded, skipping init-db"))))) 134 | 135 | #?(:clj 136 | (def delayed-connection (delay (init-db)))) 137 | 138 | #?(:clj (defonce conn @delayed-connection)) 139 | 140 | (e/def db) ; injected database ref; Electric defs are always dynamic 141 | (e/def auth-conn) 142 | 143 | 144 | 145 | ;; Queries 146 | 147 | #?(:clj 148 | (defn fetch-convo-messages [db convo-id-str] 149 | (sort-by first < (d/q '[:find ?msg-created ?msg-id ?msg-text ?msg-role 150 | :in $ ?conv-id 151 | :where 152 | [?e :conversation/id ?conv-id] 153 | [?e :conversation/messages ?msg] 154 | [?msg :message/id ?msg-id] 155 | [?msg :message/role ?msg-role] 156 | [?msg :message/text ?msg-text] 157 | [?msg :message/created ?msg-created]] 158 | db convo-id-str)))) 159 | 160 | (defn T 161 | "For debugging 162 | Input → ___ → Output 163 | | 164 | | 165 | ↓ 166 | Console" 167 | ([x] 168 | (prn x) 169 | x) 170 | ([tag x] 171 | (prn tag x) 172 | x)) 173 | 174 | #?(:clj 175 | (defn fetch-convo-messages-mapped 176 | [dh-conn convo-id] 177 | (->> (d/q '[:find (pull ?msg [*]) 178 | :in $ ?convo-id 179 | :where 180 | [?c :conversation/id ?convo-id] 181 | [?c :conversation/messages ?msg]] 182 | dh-conn 183 | convo-id) 184 | (map first) 185 | (map #(update % :message.filter/value edn/read-string)) 186 | 187 | (sort-by :message/created <) 188 | #_T))) 189 | 190 | 191 | #?(:clj 192 | (defn fetch-convo-entity-id [db convo-id] 193 | (d/q '[:find [?entity-id] 194 | :in $ ?conv-id 195 | :where 196 | [?e :conversation/id ?conv-id] 197 | [?e :conversation/entity-id ?entity-id]] 198 | db convo-id))) 199 | 200 | #?(:clj 201 | (defn fetch-user-id [user-email] 202 | (:user/id (d/pull conn '[:user/id] [:user/email user-email])))) 203 | 204 | #?(:clj 205 | (defn conversations 206 | ([db search-text] 207 | (let [convo-eids (d/q '[:find [?c ...] 208 | :in $ search-txt ?includes-fn 209 | :where 210 | [?m :message/text ?msg-text] 211 | [?c :conversation/messages ?m] 212 | [?c :conversation/topic ?topic] 213 | [?c :conversation/entity-id ?entity-id] 214 | (or-join [?msg-text ?topic] 215 | [(?includes-fn ?msg-text search-txt)] 216 | [(?includes-fn ?topic search-txt)])] 217 | db search-text kit/lowercase-includes?)] 218 | (sort-by first > (d/q '[:find ?created ?e ?conv-id ?topic ?entity-id 219 | :in $ [?e ...] 220 | :where 221 | [?e :conversation/id ?conv-id] 222 | [?e :conversation/topic ?topic] 223 | [?e :conversation/created ?created] 224 | [?c :conversation/entity-id ?entity-id]] 225 | db convo-eids)))) 226 | ([db] 227 | (sort-by first > (d/q '[:find ?created ?e ?conv-id ?topic ?entity-id 228 | :where 229 | [?e :conversation/id ?conv-id] 230 | [?e :conversation/topic ?topic] 231 | [?e :conversation/created ?created] 232 | [?c :conversation/entity-id ?entity-id] 233 | (not [?e :conversation/folder])] 234 | db))))) 235 | 236 | #?(:clj 237 | (defn conversations-in-folder [db folder-id] 238 | (sort-by first > (d/q '[:find ?created ?c ?c-id ?topic ?folder-name ?entity-id 239 | :in $ ?folder-id 240 | :where 241 | [?e :folder/id ?folder-id] 242 | [?e :folder/name ?folder-name] 243 | [?c :conversation/folder ?folder-id] 244 | [?c :conversation/id ?c-id] 245 | [?c :conversation/topic ?topic] 246 | [?c :conversation/created ?created] 247 | [?c :conversation/entity-id ?entity-id]] 248 | db folder-id)))) 249 | 250 | #?(:clj 251 | (defn folders [db] 252 | (sort-by first > (d/q '[:find ?created ?e ?folder-id ?name 253 | :where 254 | [?e :folder/id ?folder-id] 255 | [?e :folder/name ?name] 256 | [?e :folder/created ?created]] 257 | db)))) 258 | 259 | ;; 260 | 261 | ;; Transactions 262 | #?(:clj 263 | (defn transact-user-msg [conn convo-id user-query] 264 | (let [time-point (System/currentTimeMillis) 265 | tx-data {:conversation/id convo-id 266 | :conversation/messages [{:message/id (nano-id) 267 | :message/text user-query 268 | :message/role :user 269 | :message/voice :user 270 | :message/completion true 271 | :message/kind :kind/markdown 272 | :message/created time-point}]} 273 | _ (prn "transact-user-msg called for query: " user-query)] 274 | (d/transact conn [tx-data])))) 275 | 276 | #?(:clj 277 | (defn transact-assistant-msg [conn convo-id msg] 278 | (d/transact conn [{:conversation/id convo-id 279 | :conversation/messages [{:message/id (nano-id) 280 | :message/text msg 281 | :message/role :assistant 282 | :message/voice :assistant 283 | :message/completion true 284 | :message/kind :kind/markdown 285 | :message/created (System/currentTimeMillis)}]}]))) 286 | 287 | 288 | (defn transact-retrieval-prompt [conn convo-id message-id prompt] 289 | (d/transact conn [{:conversation/id convo-id 290 | :conversation/messages [{:message/id (if (nil? message-id) (nano-id) message-id) 291 | :message/text prompt 292 | :message/role :user 293 | :message/voice :agent 294 | :message/completion true 295 | :message/kind :kind/markdown 296 | :message/created (System/currentTimeMillis)}]}])) 297 | 298 | (defn transact-retrieved-sources [conn convo-id message-id formatted-msg] 299 | (d/transact conn [{:conversation/id convo-id 300 | :conversation/messages [{:message/id (if (nil? message-id) (nano-id) message-id) 301 | :message/text formatted-msg 302 | :message/role :system 303 | :message/voice :assistant 304 | :message/completion false ;; this message doesn't get sent to the llm 305 | :message/kind :kind/html 306 | :message/created (System/currentTimeMillis)}]}])) 307 | 308 | (defn create-folder [conn] 309 | (d/transact conn [{:folder/id (nano-id) 310 | :folder/name "New folder" 311 | :folder/created (System/currentTimeMillis)}])) 312 | 313 | (defn rename-convo-topic [conn convo-id new-topic] 314 | (d/transact conn [{:db/id [:conversation/id convo-id] 315 | :conversation/topic new-topic}])) 316 | 317 | (defn rename-folder [conn folder-id new-folder-name] 318 | (d/transact conn [{:db/id [:folder/id folder-id] 319 | :folder/name new-folder-name}])) 320 | 321 | (defn delete-convo [conn convo-eid] 322 | (d/transact conn [[:db/retract convo-eid :conversation/id]])) ; TODO: develop consistency of id and eid usage 323 | 324 | (defn delete-folder [conn folder-eid] 325 | (d/transact conn [[:db.fn/retractEntity folder-eid]])) 326 | 327 | (defn clear-all-conversations [conn] 328 | (let [convo-eids (map :e (d/datoms @conn :avet :conversation/id)) 329 | folder-eids (map :e (d/datoms @conn :avet :folder/id)) 330 | m-eids (set (map first (d/q '[:find ?m 331 | :in $ [?convo-id ...] 332 | :where 333 | [?convo-id :conversation/messages ?m]] @conn convo-eids))) 334 | retraction-ops (concat 335 | (mapv (fn [eid] [:db.fn/retractEntity eid :conversation/id]) convo-eids) 336 | (mapv (fn [eid] [:db.fn/retractEntity eid :folder/id]) m-eids) 337 | (mapv (fn [eid] [:db.fn/retractEntity eid :folder/id]) folder-eids))] 338 | (d/transact conn retraction-ops))) 339 | 340 | #?(:clj 341 | (defn transact-new-msg-thread [conn entity-id] 342 | (let [convo-id (nano-id) 343 | time-point (System/currentTimeMillis) 344 | tx-data 345 | {:conversation/id convo-id 346 | :conversation/entity-id entity-id 347 | :conversation/topic "Ny samtale" 348 | :conversation/created time-point 349 | :conversation/system-prompt "sys-prompt" 350 | :conversation/messages [{:message/id (nano-id) 351 | :message/text "You are a helpful assistant." 352 | :message/role :system 353 | :message/voice :agent 354 | :message/completion true 355 | :message/kind :kind/text 356 | :message/created time-point} 357 | {:message/id (nano-id) 358 | :message/voice :filter 359 | :message/created (inc time-point) 360 | :message/completion false 361 | :message.filter/value 362 | (pr-str {:type :typesense 363 | :fields [{:type :multiselect 364 | :expanded? true 365 | :selected-options #{} 366 | :field "type"} 367 | {:type :multiselect 368 | :expanded? true 369 | :selected-options #{} 370 | :field "orgs_long"} 371 | #_{:type :multiselect 372 | :expanded? true 373 | :selected-options #{} 374 | :field "source_published_year"}]} 375 | 376 | )}]} 377 | _ (prn "transact-new-msg-thread called" )] 378 | (d/transact conn [tx-data]) 379 | {:conversation-id convo-id}))) 380 | 381 | 382 | #?(:clj 383 | (defn set-message-filter [conn id new-message-filter] 384 | (prn [new-message-filter]) 385 | (d/transact conn [{:db/id id 386 | :message.filter/value (pr-str new-message-filter)}]) 387 | nil)) 388 | 389 | (comment 390 | 391 | ;; Helpers 392 | 393 | (defn ->eid-action [data action-fn] 394 | (let [eid (second (first data))] 395 | (when eid 396 | (action-fn eid)))) 397 | 398 | (defn ->id-action [data action-fn] 399 | (let [eid (nth (first data) 2)] 400 | (when eid 401 | (action-fn eid)))) 402 | 403 | ;; Creation 404 | 405 | ;Create folder 406 | (create-folder conn) 407 | 408 | ; Existing conversations 409 | ; Ensure that the convo-id exists in the db 410 | (transact-user-msg {:convo-id "test-2" 411 | :msg "hello world"}) 412 | ; New conversation 413 | (transact-user-msg {:new-convo? true 414 | :convo-id (nano-id) 415 | :msg "test message"}) 416 | 417 | ; New conversation 418 | (transact-new-msg-thread conn {:convo-id (nano-id) 419 | :user-query "hvilke kategorier gjelder for KI system risiko?" 420 | :entity-id "7i8dadbe-0101-f0e1-92b8-40b10a61cdcd" ;; AI Guide 421 | }) 422 | 423 | 424 | ;; Queries 425 | (conversations @conn) 426 | (folders @conn) 427 | 428 | ;; Update 429 | 430 | ; rename conversation topic 431 | (-> (conversations @conn) 432 | (->id-action #(rename-convo-topic conn % "new convo name"))) 433 | 434 | ; rename folder 435 | (-> (folders @conn) 436 | (->id-action #(rename-folder conn % "Updated folder name"))) 437 | 438 | ;; Deletions 439 | 440 | (-> (conversations @conn) 441 | (->eid-action #(delete-convo conn %))) 442 | 443 | (-> (folders @conn) 444 | (->eid-action #(delete-convo conn %))) 445 | 446 | (clear-all-conversations conn) 447 | 448 | ;; TODO 449 | ;(conversations-in-folder @conn ) 450 | 451 | ;; Database migration scripts 452 | (require '[datahike.migrate :refer [export-db import-db]]) 453 | 454 | (def local-cfg (get-in (config) [:db :local])) 455 | (def remote-cfg (get-in (config) [:db :remote])) 456 | 457 | (def create-local-db (when-not (d/database-exists? local-cfg) (d/create-database local-cfg))) 458 | 459 | (def local-conn (d/connect local-cfg)) 460 | (def remote-conn (d/connect remote-cfg)) 461 | 462 | (conversations @local-conn) 463 | (conversations @remote-conn) 464 | 465 | (export-db remote-conn "/tmp/eavt-dump") 466 | (import-db local-conn "/tmp/eavt-dump") 467 | 468 | ;; End comment 469 | ) 470 | 471 | 472 | --------------------------------------------------------------------------------