├── .dockerignore ├── .editorconfig ├── .github ├── renovate.json └── workflows │ ├── ci.yaml │ ├── release-build.yml │ └── unstable-build.yml ├── .gitignore ├── .golangci.yml ├── .vscode ├── extensions.json ├── launch.json └── settings.json ├── README.md ├── apis ├── jsonapi │ ├── jsonapi.pb.go │ └── jsonapi.proto └── llmgapi │ └── v1 │ └── openai │ ├── service.pb.go │ ├── service.proto │ └── service_grpc.pb.go ├── buf.gen.yaml ├── buf.lock ├── buf.yaml ├── cmd ├── lingticio │ ├── llmg-graphql │ │ ├── Dockerfile │ │ └── main.go │ └── llmg-grpc │ │ ├── Dockerfile │ │ └── main.go └── tools │ └── openapiv2conv │ └── main.go ├── config └── config.example.yaml ├── cspell.config.yaml ├── docker-compose.yml ├── docs └── designs │ └── 1 - Configs.md ├── go.mod ├── go.sum ├── graph └── openai │ ├── chat.graphqls │ └── gqlgen.yml ├── hack ├── proto-export └── proto-gen ├── internal ├── configs │ ├── common.go │ ├── configs.go │ ├── llmg.go │ └── types.go ├── constants │ ├── cookie.go │ ├── metadata.go │ └── sizes.go ├── datastore │ ├── cache.go │ ├── datastore.go │ └── rueidis.go ├── graph │ ├── openai │ │ ├── generated │ │ │ └── generated.go │ │ ├── model │ │ │ └── models_gen.go │ │ ├── openai.go │ │ └── resolvers │ │ │ ├── chat.resolver.go │ │ │ ├── common.go │ │ │ └── resolver.go │ └── server │ │ ├── middlewares │ │ └── headers.go │ │ └── server.go ├── grpc │ ├── servers │ │ ├── apiserver │ │ │ ├── grpc_gateway.go │ │ │ └── grpc_server.go │ │ ├── interceptors │ │ │ ├── apikey.go │ │ │ ├── authorization.go │ │ │ ├── cookie.go │ │ │ ├── error_handler.go │ │ │ ├── panic.go │ │ │ └── request_path.go │ │ ├── llmg │ │ │ └── v1 │ │ │ │ └── v1.go │ │ ├── middlewares │ │ │ ├── response_log.go │ │ │ └── route_not_found.go │ │ └── servers.go │ └── services │ │ ├── llmgapi │ │ └── v1 │ │ │ └── openai │ │ │ ├── common.go │ │ │ └── openai.go │ │ └── services.go ├── libs │ ├── libs.go │ └── logger.go ├── meta │ └── meta.go └── types │ └── redis │ └── rediskeys │ └── keys.go ├── pkg ├── apierrors │ ├── apierrors.go │ └── errors.go ├── configs │ └── cfgproviders │ │ ├── auth.go │ │ ├── rds_auth.go │ │ ├── rds_auth_test.go │ │ ├── redis_auth.go │ │ ├── redis_auth_test.go │ │ ├── static_auth.go │ │ └── static_auth_test.go ├── neuri │ └── formats │ │ └── jsonfmt │ │ ├── json_parser.go │ │ ├── json_parser_test.go │ │ ├── jsonschema.go │ │ └── jsonschema_test.go ├── semanticcache │ └── rueidis │ │ ├── rueidis.go │ │ └── rueidis_test.go ├── types │ ├── jwtclaim │ │ └── jwtclaim.go │ ├── metadata │ │ └── metadata.go │ └── redis │ │ └── rediskeys │ │ └── keys.go └── util │ ├── eventsource │ └── eventsource.go │ ├── grpc │ ├── gateway.go │ └── register.go │ ├── nanoid │ └── nanoid.go │ └── utils │ ├── file.go │ ├── path.go │ ├── string.go │ └── unittest.go └── tools └── tools.go /.dockerignore: -------------------------------------------------------------------------------- 1 | # Folders 2 | .git 3 | .github/ 4 | .postgres/ 5 | docs/ 6 | 7 | # Files 8 | .dockerignore 9 | .gitignore 10 | docker-compose.yml 11 | Dockerfile 12 | README.md 13 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*.proto] 4 | indent_size = 2 5 | indent_style = space 6 | insert_final_newline = true 7 | 8 | [*.yaml] 9 | indent_size = 2 10 | indent_style = space 11 | insert_final_newline = true 12 | 13 | [*.go] 14 | charset = utf-8 15 | indent_style = tab 16 | indent_size = 4 17 | end_of_line = lf 18 | insert_final_newline = true 19 | trim_trailing_whitespace = true 20 | -------------------------------------------------------------------------------- /.github/renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": [ 4 | "config:recommended", 5 | ":dependencyDashboard", 6 | ":semanticPrefixFixDepsChoreOthers", 7 | ":prHourlyLimitNone", 8 | ":prConcurrentLimitNone", 9 | ":ignoreModulesAndTests", 10 | "schedule:monthly", 11 | "group:allNonMajor", 12 | "replacements:all", 13 | "workarounds:all" 14 | ] 15 | } 16 | -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | branches: 9 | - main 10 | 11 | env: 12 | STORE_PATH: "" 13 | 14 | jobs: 15 | build-test: 16 | name: Build Test 17 | runs-on: "ubuntu-latest" 18 | 19 | steps: 20 | - name: Checkout 21 | uses: actions/checkout@v4 22 | 23 | - uses: actions/setup-go@v5 24 | with: 25 | go-version: "^1.23" 26 | cache: true 27 | 28 | - name: Setup Go Cache PATH 29 | id: go-cache-paths 30 | run: | 31 | echo "go-build=$(go env GOCACHE)" >> $GITHUB_OUTPUT 32 | echo "go-mod=$(go env GOMODCACHE)" >> $GITHUB_OUTPUT 33 | 34 | - name: Go Build Cache 35 | uses: actions/cache@v4 36 | with: 37 | path: ${{ steps.go-cache-paths.outputs.go-build }} 38 | key: ${{ runner.os }}-go-build-${{ hashFiles('**/go.sum') }} 39 | 40 | - name: Go Mod Cache 41 | uses: actions/cache@v4 42 | with: 43 | path: ${{ steps.go-cache-paths.outputs.go-mod }} 44 | key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }} 45 | 46 | - name: Test Build 47 | run: go build ./... 48 | 49 | lint: 50 | name: Lint 51 | runs-on: ubuntu-latest 52 | steps: 53 | - uses: actions/setup-go@v5 54 | with: 55 | go-version: "^1.23" 56 | cache: true 57 | 58 | - uses: actions/checkout@v4 59 | - name: golangci-lint 60 | uses: golangci/golangci-lint-action@v3.4.0 61 | with: 62 | # Optional: golangci-lint command line arguments. 63 | version: v1.62.0 64 | args: "--timeout=10m" 65 | 66 | unittest: 67 | name: Unit Test 68 | runs-on: ubuntu-latest 69 | 70 | services: 71 | # Label used to access the service container 72 | postgres: 73 | # Docker Hub image 74 | image: postgres 75 | # Provide the password for postgres 76 | env: 77 | POSTGRES_PASSWORD: "123456" 78 | # Set health checks to wait until postgres has started 79 | options: >- 80 | --health-cmd pg_isready 81 | --health-interval 10s 82 | --health-timeout 5s 83 | --health-retries 5 84 | ports: 85 | # Maps tcp port 5432 on service container to the host 86 | - 5432:5432 87 | # Label used to access the service container 88 | redis: 89 | # Docker Hub image 90 | image: redis 91 | # Set health checks to wait until redis has started 92 | options: >- 93 | --health-cmd "redis-cli ping" 94 | --health-interval 10s 95 | --health-timeout 5s 96 | --health-retries 5 97 | ports: 98 | # Maps tcp port 6379 on service container to the host 99 | - 6379:6379 100 | 101 | steps: 102 | - uses: actions/checkout@v4 103 | 104 | - name: Setup Go 105 | uses: actions/setup-go@v5 106 | with: 107 | go-version: "^1.23" 108 | cache: true 109 | 110 | # Get values for cache paths to be used in later steps 111 | - name: Setup Go Cache PATH 112 | id: go-cache-paths 113 | run: | 114 | echo "go-build=$(go env GOCACHE)" >> $GITHUB_OUTPUT 115 | echo "go-mod=$(go env GOMODCACHE)" >> $GITHUB_OUTPUT 116 | 117 | # Cache go build cache, used to speedup go test 118 | - name: Go Build Cache 119 | uses: actions/cache@v4 120 | with: 121 | path: ${{ steps.go-cache-paths.outputs.go-build }} 122 | key: ${{ runner.os }}-go-build-${{ hashFiles('**/go.sum') }} 123 | 124 | # Cache go mod cache, used to speedup builds 125 | - name: Go Mod Cache 126 | uses: actions/cache@v4 127 | with: 128 | path: ${{ steps.go-cache-paths.outputs.go-mod }} 129 | key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }} 130 | 131 | - name: Unit tests 132 | run: | 133 | go test ./... -coverprofile=coverage.out -covermode=atomic -p=1 134 | go tool cover -func coverage.out 135 | env: 136 | OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} 137 | OPENAI_API_BASEURL: ${{ secrets.OPENAI_API_BASEURL }} 138 | -------------------------------------------------------------------------------- /.github/workflows/release-build.yml: -------------------------------------------------------------------------------- 1 | name: Release Build 2 | 3 | on: 4 | push: 5 | tags: 6 | - "**" 7 | workflow_dispatch: 8 | 9 | jobs: 10 | hub_build: 11 | name: Build for Docker Hub 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v4 15 | 16 | - name: Fetch version 17 | id: version 18 | run: | 19 | export LAST_TAGGED_COMMIT=$(git rev-list --tags --max-count=1) 20 | export LAST_TAG=$(git describe --tags $LAST_TAGGED_COMMIT) 21 | echo "version=${LAST_TAG#v}" >> $GITHUB_OUTPUT 22 | 23 | - name: Set up Docker Buildx 24 | uses: docker/setup-buildx-action@v3 25 | with: 26 | platforms: linux/amd64,linux/arm64 27 | 28 | - name: Sign in to Docker Hub 29 | uses: docker/login-action@v3 30 | with: 31 | username: ${{ github.repository_owner }} 32 | password: ${{ secrets.DOCKER_ACCESS_TOKEN }} 33 | 34 | - name: Create image tags 35 | id: dockerinfo 36 | run: | 37 | echo "taglatest=${{ github.repository }}:latest" >> $GITHUB_OUTPUT 38 | echo "tag=${{ github.repository }}:${{ steps.version.outputs.version }}" >> $GITHUB_OUTPUT 39 | 40 | - name: Build and Push 41 | uses: docker/build-push-action@v6 42 | with: 43 | context: ./ 44 | file: ./Dockerfile 45 | push: true 46 | no-cache: false 47 | tags: | 48 | ${{ steps.dockerinfo.outputs.taglatest }} 49 | ${{ steps.dockerinfo.outputs.tag }} 50 | -------------------------------------------------------------------------------- /.github/workflows/unstable-build.yml: -------------------------------------------------------------------------------- 1 | name: Unstable Build 2 | 3 | on: 4 | workflow_dispatch: 5 | 6 | jobs: 7 | hub_build: 8 | name: Build for Docker Hub 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v4 12 | 13 | - name: Set up Docker Buildx 14 | uses: docker/setup-buildx-action@v3 15 | with: 16 | platforms: linux/amd64,linux/arm64 17 | 18 | - name: Sign in to Docker Hub 19 | uses: docker/login-action@v3 20 | with: 21 | username: ${{ github.repository_owner }} 22 | password: ${{ secrets.DOCKER_ACCESS_TOKEN }} 23 | 24 | - name: Create image tags 25 | id: dockerinfo 26 | run: | 27 | echo "tagunstable=${{ github.repository }}:unstable" >> $GITHUB_OUTPUT 28 | 29 | - name: Build and Push 30 | uses: docker/build-push-action@v6 31 | with: 32 | context: ./ 33 | file: ./Dockerfile 34 | push: true 35 | no-cache: false 36 | tags: | 37 | ${{ steps.dockerinfo.outputs.tagunstable }} 38 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # IDE 2 | .idea/ 3 | 4 | # Created by https://www.gitignore.io/api/visualstudiocode 5 | # Edit at https://www.gitignore.io/?templates=visualstudiocode 6 | 7 | ### VisualStudioCode ### 8 | # Maybe .vscode/**/* instead - see comments 9 | .vscode/* 10 | !.vscode/settings.json 11 | !.vscode/tasks.json 12 | !.vscode/launch.json 13 | !.vscode/extensions.json 14 | 15 | ### VisualStudioCode Patch ### 16 | # Ignore all local history of files 17 | **/.history 18 | 19 | # End of https://www.gitignore.io/api/visualstudiocode 20 | 21 | # Build / Release 22 | *.exe 23 | *.exe~ 24 | *.dll 25 | *.so 26 | *.dylib 27 | *.db 28 | *.bin 29 | *.tar.gz 30 | /release/ 31 | 32 | # Runtime / Compile Temporary Assets 33 | vendor/ 34 | logs/ 35 | 36 | # Credentials 37 | cert*/ 38 | *.pem 39 | *.crt 40 | *.cer 41 | *.key 42 | *.p12 43 | 44 | # Test binary, build with `go test -c` 45 | *.test 46 | cover* 47 | coverage* 48 | 49 | # Output of the go coverage tool, specifically when used with LiteIDE 50 | *.out 51 | 52 | # macOS 53 | .DS_Store 54 | 55 | # Configurations 56 | config.yaml 57 | config.yml 58 | .env 59 | 60 | # Local Configuration 61 | config.local.yaml 62 | 63 | # Temporary 64 | temp/ 65 | .cache 66 | .temp 67 | 68 | # Local pgSQL db 69 | .postgres/ 70 | 71 | # Debug 72 | __debug* 73 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | linters: 2 | enable-all: true 3 | disable: 4 | - depguard 5 | - exportloopref 6 | - execinquery 7 | - gomnd 8 | - funlen 9 | - containedctx 10 | - exhaustruct 11 | - testpackage 12 | - varnamelen 13 | - maintidx 14 | - err113 15 | - nlreturn 16 | - wrapcheck 17 | - tagliatelle 18 | - paralleltest 19 | - lll 20 | - contextcheck 21 | - gochecknoglobals 22 | - tagalign 23 | - nilnil 24 | - godot 25 | - godox 26 | - gci 27 | - gocognit 28 | - gocyclo 29 | - cyclop 30 | - ireturn 31 | - gofumpt 32 | 33 | linters-settings: 34 | wsl: 35 | allow-assign-and-call: false 36 | strict-append: false 37 | revive: 38 | rules: 39 | - name: blank-imports 40 | disabled: true 41 | nestif: 42 | # Minimal complexity of if statements to report. 43 | # Default: 5 44 | min-complexity: 9 45 | dupl: 46 | # Tokens count to trigger issue. 47 | # Default: 150 48 | threshold: 600 49 | mnd: 50 | ignored-functions: 51 | - "context.WithTimeout" 52 | gocritic: 53 | disabled-checks: 54 | - ifElseChain 55 | 56 | issues: 57 | exclude: 58 | - "if statements should only be cuddled with assignments" # from wsl 59 | - "if statements should only be cuddled with assignments used in the if statement itself" # from wsl 60 | - "assignments should only be cuddled with other assignments" # from wsl. false positive case: var a bool\nb := true 61 | - "declarations should never be cuddled" # from wsl 62 | exclude-rules: 63 | - path: _test\.go 64 | linters: 65 | - perfsprint 66 | exclude-dirs: 67 | - ent 68 | - apis 69 | -------------------------------------------------------------------------------- /.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": [ 3 | "streetsidesoftware.code-spell-checker", 4 | "mikestead.dotenv", 5 | "EditorConfig.EditorConfig", 6 | "yzhang.markdown-all-in-one", 7 | "redhat.vscode-yaml", 8 | "golang.go", 9 | "bufbuild.vscode-buf", 10 | "zxh404.vscode-proto3", 11 | "GraphQL.vscode-graphql-execution", 12 | "GraphQL.vscode-graphql", 13 | "GraphQL.vscode-graphql-syntax" 14 | ] 15 | } 16 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "Launch llmg GraphQL", 9 | "type": "go", 10 | "request": "launch", 11 | "mode": "auto", 12 | "program": "${workspaceFolder}/cmd/lingticio/llmg-graphql/main.go", 13 | "args": [ 14 | "-e", 15 | "${workspaceFolder}/cmd/lingticio/llmg-graphql/.env", 16 | ] 17 | }, 18 | { 19 | "name": "Launch llmg gRPC", 20 | "type": "go", 21 | "request": "launch", 22 | "mode": "auto", 23 | "program": "${workspaceFolder}/cmd/lingticio/llmg-grpc/main.go", 24 | "args": [ 25 | "-e", 26 | "${workspaceFolder}/cmd/lingticio/llmg-grpc/.env", 27 | ] 28 | } 29 | ] 30 | } 31 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "go.useLanguageServer": true, 3 | // "go.buildOnSave": "workspace", 4 | "go.lintOnSave": "package", 5 | "go.vetOnSave": "workspace", 6 | "go.coverOnSave": false, 7 | "go.lintTool": "golangci-lint", 8 | "go.lintFlags": [ 9 | "--config=${workspaceFolder}/.golangci.yml", 10 | ], 11 | "go.inferGopath": true, 12 | "go.alternateTools": {}, 13 | "go.coverOnSingleTest": true, 14 | "go.testTimeout": "900s", 15 | "go.testFlags": [ 16 | "-v", 17 | "-count=1" 18 | ], 19 | "go.toolsManagement.autoUpdate": true, 20 | "go.coverOnSingleTestFile": true, 21 | "gopls": { 22 | "build.buildFlags": [], 23 | "ui.completion.usePlaceholders": true, 24 | "ui.semanticTokens": true 25 | }, 26 | "[go]": { 27 | "editor.insertSpaces": false, 28 | "editor.formatOnSave": true, 29 | "editor.codeActionsOnSave": { 30 | "source.organizeImports": "always" 31 | }, 32 | }, 33 | // Consider integration with Buf · Issue #138 · zxh0/vscode-proto3 34 | // https://github.com/zxh0/vscode-proto3/issues/138 35 | "protoc": { 36 | "options": [ 37 | "--proto_path=${workspaceFolder}/.temp/cache/buf.build/vendor/proto", 38 | ] 39 | }, 40 | } 41 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # llmg 2 | 3 | 🧘 Extensive LLM endpoints, expended capabilities through your favorite protocols, 🕸️ GraphQL, ↔️ gRPC, ♾️ WebSocket. Extended SOTA support for structured data, function calling, instruction mapping, load balancing, grouping, intelli-routing. Advanced tracing and inference tracking. 4 | 5 | ## Features 6 | 7 | - [ ] Grouping 8 | - [ ] Intelli-routing 9 | - [ ] Load balancing 10 | - [ ] Semantic cache 11 | - [x] Structured data 12 | - [x] Powered by Neuri 13 | - [x] JSON 14 | - [x] Support any order of structured data 15 | - [x] JSON streaming support 16 | - [x] JSON tokenizer & parser 17 | - [ ] Code snippet 18 | - [ ] WASM with vscode-textmate 19 | - [ ] Function calling 20 | - [ ] Instruction mapping 21 | - [ ] Generative streaming 22 | - [ ] Advanced tracing 23 | - [ ] Inference tracking 24 | 25 | ## Providers 26 | 27 | - [x] OpenAI 28 | 29 | ## Protocols 30 | 31 | - [x] GraphQL 32 | - [x] gRPC 33 | - [ ] WebSockets 34 | - [ ] RESTful 35 | 36 | ## Project structure 37 | 38 | ``` 39 | . 40 | ├── apis # Protobuf files 41 | │ ├── jsonapi # Shared definitions 42 | │ └── gatewayapi # Business APIs of Gateway 43 | ├── cmd # Entry for microservices 44 | ├── config # Configuration files 45 | ├── graph # GraphQL Schemas, gqlgen configurations 46 | ├── hack # Scripts for both development, testing, deployment 47 | ├── internal # Actual implementation 48 | │ ├── configs # Configuration 49 | │ ├── constants # Constants 50 | │ ├── graph # GraphQL server & model & resolvers 51 | │ ├── grpc # gRPC server and client 52 | │ ├── libs # Libraries 53 | │ └── meta # Metadata 54 | ├── logs # Logs, excluded from git 55 | ├── pkg # Public APIs 56 | ├── production # Production related deployment, configurations and scripts 57 | ├── release # Release bundles, excluded from git 58 | ├── tools # Tools 59 | │ └── tools.go # Pinned tools 60 | ├── .dockerignore # Docker ignore file 61 | ├── .editorconfig # Editor configuration 62 | ├── .gitignore # Git ignore file 63 | ├── .golangci.yml # GolangCI configuration 64 | ├── buf.gen.yaml # How .proto under apis/ are generated 65 | ├── buf.yaml # How buf is configured 66 | ├── cspell.config.yaml # CSpell configuration 67 | └── docker-compose.yml # Docker compose file, for bootstrapping the needed external services like db, redis, etc. 68 | ``` 69 | 70 | ## Stacks involved 71 | 72 | - [Go](https://golang.org/) 73 | - [gqlgen](https://gqlgen.com/) 74 | - [gRPC](https://grpc.io/) 75 | - [uber/zap](https://go.uber.org/zap) 76 | - [uber/fx](https://go.uber.org/fx) 77 | - [Docker](https://docker.io/) 78 | - [Grafana Promtail](https://grafana.com/docs/loki/latest/send-data/promtail/) 79 | - [Buf](https://buf.build/) 80 | 81 | ## Configuration 82 | 83 | Copy the `config.example.yaml` to `config.yaml` and update the values as needed. 84 | 85 | ```shell 86 | cp config.example.yaml config.yaml 87 | ``` 88 | 89 | > [!NOTE] 90 | > When developing locally, you can use the `config.local.yaml` file to override both testing and production configurations without need to worry 91 | > about accidentally committing sensitive information since it is ignored by git. 92 | 93 | Besides configurations, you can always use environment variables to override the configurations as well. 94 | 95 | ## Build 96 | 97 | Every microservice and its entry should have similar build steps and usage as follows. 98 | 99 | ### Build `llmg-grpc` 100 | 101 | ```shell 102 | go build \ 103 | -a \ 104 | -o "release/lingticio/llmg-grpc" \ 105 | -ldflags " -X './internal/meta.Version=1.0.0' -X './internal/meta.LastCommit=abcdefg'" \ 106 | "./cmd/lingticio/llmg-grpc" 107 | ``` 108 | 109 | ### Build `llmg-grpc` with Docker 110 | 111 | ```shell 112 | docker build \ 113 | --build-arg="BUILD_VERSION=1.0.0" \ 114 | --build-arg="BUILD_LAST_COMMIT=abcdefg" \ 115 | -f cmd/lingticio/llmg-grpc/Dockerfile \ 116 | . 117 | ``` 118 | 119 | ## Start the server 120 | 121 | ### Start `llmg-grpc` 122 | 123 | With `config/config.yaml`: 124 | 125 | ```shell 126 | go run cmd/lingticio/llmg-grpc 127 | ``` 128 | 129 | With `config.local.yaml`: 130 | 131 | ```shell 132 | go run cmd/lingticio/llmg-grpc -c $(pwd)/config/config.local.yaml 133 | ``` 134 | 135 | ## Development 136 | 137 | ### Adding new queries, mutations, or subscriptions for GraphQL 138 | 139 | We use [`gqlgen`](https://gqlgen.com/) to generate the GraphQL server and client codes based on the schema defined in the `graph/${category}/*.graphqls` file. 140 | 141 | #### Generate the GraphQL server and client codes 142 | 143 | ```shell 144 | go generate ./... 145 | ``` 146 | 147 | Once generated, you can start the server and test the queries, mutations, and subscriptions from `internal/graph/${category}/*.resolvers.go`. 148 | 149 | ### Prepare buf.build Protobuf dependencies 150 | 151 | ```shell 152 | buf dep update 153 | chmod +x ./hack/proto-export 154 | ./hack/proto-export 155 | ``` 156 | 157 | ### Adding new services or endpoints 158 | 159 | We use [`buf`](https://buf.build/) to manage and generate the APIs based on the Protobuf files. 160 | 161 | #### Install `buf` 162 | 163 | Follow the instructions here: [Buf - Install the Buf CLI](https://buf.build/docs/installation) 164 | 165 | #### Prepare `buf` 166 | 167 | ```shell 168 | buf dep update 169 | ``` 170 | 171 | #### Create new Protobuf files 172 | 173 | Create new Protobuf files under the `apis` directory as following guidelines: 174 | 175 | ``` 176 | . 177 | apis 178 | ├── jsonapi # 179 | │ └── jsonapi.proto 180 | └── coreapi # 181 | └── v1 # 182 | └── v1.proto 183 | ``` 184 | 185 | #### Generate the APIs 186 | 187 | ##### Install `grpc-ecosystem/grpc-gateway-ts` plugin 188 | 189 | ```shell 190 | go install github.com/grpc-ecosystem/protoc-gen-grpc-gateway-ts 191 | ``` 192 | 193 | Run the following command to generate the needed files: 194 | 195 | ```shell 196 | buf generate 197 | ``` 198 | 199 | The generated files includes: 200 | 201 | 1. `*.gw.go` files for gRPC-Gateway 202 | 2. `*.pb.go` files for gRPC server and client 203 | 3. `*.swagger.json` files for Swagger UI 204 | 205 | Then you are off to go. 206 | 207 | ## [Adding new Test Doubles (a.k.a. Mocks)](https://github.com/maxbrunsfeld/counterfeiter) 208 | 209 | To test the gRPC clients and all sorts of objects like this, as well as meet the [SOLID](https://en.wikipedia.org/wiki/SOLID) principle, we use a library called [counterfeiter](https://github.com/maxbrunsfeld/counterfeiter) to generate test doubles and quickly mock out the dependencies for both local defined and third party interfaces. 210 | 211 | Generally all the generated test doubles are generated under the `fake` directory that located in the same package as the original interface. 212 | 213 | #### Update the existing test doubles 214 | 215 | After you have updated the interface, you can run the following command to update and generate the test doubles again freshly: 216 | 217 | ```bash 218 | go generate ./... 219 | ``` 220 | 221 | #### Generate new test doubles for new interfaces 222 | 223 | First you need to ensure the following comment annotation has been added to the package where you hold all the initial references to the interfaces in order to make sure the `go generate` command can find the interfaces: 224 | 225 | ```go 226 | //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate 227 | ``` 228 | 229 | If the above comment annotation hasn't yet been added, add one please. 230 | 231 | Then you can add the following comment annotation to tell counterfeiter to generate the test doubles for the interface: 232 | 233 | ##### Generate new test doubles for local defined interfaces 234 | 235 | ```go 236 | //counterfeiter:generate -o --fake-name 237 | ``` 238 | 239 | For example: 240 | 241 | ```go 242 | //counterfeiter:generate -o fake/some_client.go --fake-name FakeClient . Client 243 | type Client struct { 244 | Method() string 245 | } 246 | ``` 247 | 248 | ##### Generate new test doubles for third party interfaces 249 | 250 | ```go 251 | //counterfeiter:generate -o --fake-name 252 | ``` 253 | 254 | For example: 255 | 256 | ```go 257 | import ( 258 | "github.com/some/package" 259 | ) 260 | 261 | //counterfeiter:generate -o fake/some_client.go --fake-name FakeClient github.com/some/package.Client 262 | 263 | var ( 264 | client package.Client 265 | ) 266 | ``` 267 | 268 | ## Acknowledgement 269 | 270 | - Heavily inspired by [Portkey.ai](https://portkey.ai/) 271 | - Heavily inspired by [AI Gateway - LLM API Solution | Kong Inc.](https://konghq.com/products/kong-ai-gateway) 272 | - Heavily inspired by [Unify: The Best LLM on Every Prompt](https://unify.ai/) 273 | - [lm-sys/RouteLLM: A framework for serving and evaluating LLM routers - save LLM costs without compromising quality!](https://github.com/lm-sys/RouteLLM) 274 | 275 | ## Star History 276 | 277 | [![Star History Chart](https://api.star-history.com/svg?repos=lingticio/llmg&type=Date)](https://star-history.com/#lingticio/llmg&Date) 278 | 279 | ## Contributors 280 | 281 | Thanks to everyone who contributed to the this project! 282 | 283 | [![contributors](https://contrib.rocks/image?repo=lingticio/llmg)](https://github.com/lingticio/llmg/graphs/contributors) 284 | 285 | ### Written with ♥ 286 | -------------------------------------------------------------------------------- /apis/jsonapi/jsonapi.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package apis.jsonapi; 4 | 5 | import "google/protobuf/any.proto"; 6 | import "protoc-gen-openapiv2/options/annotations.proto"; 7 | 8 | option go_package = "github.com/lingticio/llmg/apis/jsonapi"; 9 | 10 | // Where specified, a links member can be used to represent links. 11 | message Links { 12 | // a string whose value is a URI-reference [RFC3986 Section 4.1] pointing to the link’s target. 13 | string href = 1 [(grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = {example: "\"https://apidocs.example.com/errors/BAD_REQUEST\""}]; 14 | // a string indicating the link’s relation type. 15 | optional string rel = 2 [(grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = {example: "\"external\""}]; 16 | // a link to a description document (e.g. OpenAPI or JSON Schema) for the link target. 17 | optional string describedby = 3 [(grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = {example: "\"OpenAPI\""}]; 18 | // a string which serves as a label for the destination of a link 19 | // such that it can be used as a human-readable identifier (e.g., a menu entry). 20 | optional string title = 4 [(grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = {example: "\"Learn more about BAD_REQUEST\""}]; 21 | // a string indicating the media type of the link’s target. 22 | optional string type = 5 [(grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = {example: "\"text/html\""}]; 23 | // a string or an array of strings indicating the language(s) of the link’s target. 24 | // An array of strings indicates that the link’s target is available in multiple languages. 25 | optional string hreflang = 6 [(grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = {example: "\"en-US\""}]; 26 | // a meta object containing non-standard meta-information about the link. 27 | map meta = 7; 28 | } 29 | 30 | message Response { 31 | // Data is the primary data for a response. 32 | repeated google.protobuf.Any data = 1; 33 | // Errors is an array of error objects. 34 | repeated ErrorObject errors = 2; 35 | } 36 | 37 | message ErrorObjectSource { 38 | // a JSON Pointer [RFC6901] to the value in the request document that caused the error 39 | // [e.g. "/data" for a primary data object, or "/data/attributes/title" for a specific attribute]. 40 | string pointer = 1 [(grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = {example: "\"/user/age\""}]; 41 | // a string indicating which URI query parameter caused the error. 42 | string parameter = 2 [(grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = {example: "\"created_at.ASC\""}]; 43 | // a string indicating the name of a single request header which caused the error. 44 | string header = 3 [(grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = {example: "\"X-SOME-HEADER\""}]; 45 | } 46 | 47 | message ErrorObject { 48 | // a unique identifier for this particular occurrence of the problem. 49 | string id = 1 [(grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = {example: "\"BAD_REQUEST\""}]; 50 | // a links object containing references to the source of the error. 51 | optional Links links = 2; 52 | // the HTTP status code applicable to this problem, expressed as a string value. 53 | uint64 status = 3 [(grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = {example: "\"400\""}]; 54 | // an application-specific error code, expressed as a string value. 55 | string code = 4 [(grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = {example: "\"USER_UPDATE_FAILED\""}]; 56 | // a short, human-readable summary of the problem 57 | string title = 5 [(grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = {example: "\"Failed to update user's profile, invalid parameter(s) detected\""}]; 58 | // a human-readable explanation specific to this occurrence of the problem. Like title. 59 | string detail = 6 [(grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = {example: "\"A field under /user/age is not correct, should be 'number' instead of 'string'\""}]; 60 | // an object containing references to the source of the error. 61 | optional ErrorObjectSource source = 7; 62 | // a meta object containing non-standard meta-information about the error. 63 | map meta = 8; 64 | } 65 | 66 | message ErrorCaller { 67 | string file = 1; 68 | int64 line = 2; 69 | string function = 3; 70 | } 71 | 72 | // pageInfo is used to indicate whether more edges exist prior or following the set defined by the clients arguments. 73 | message PageInfo { 74 | // hasPreviousPage is used to indicate whether more edges exist prior to the set defined by the clients arguments. 75 | // If the client is paginating with last/before, then the server must return true if prior edges exist, otherwise false. 76 | // If the client is paginating with first/after, then the client may return true if edges prior to after exist, 77 | // if it can do so efficiently, otherwise may return false. 78 | bool has_previous_page = 1; 79 | // hasNextPage is used to indicate whether more edges exist following the set defined by the clients arguments. 80 | // If the client is paginating with first/after, then the server must return true if further edges exist, otherwise false. 81 | // If the client is paginating with last/before, then the client may return true if edges further from before exist, 82 | // if it can do so efficiently, otherwise may return false. 83 | bool has_next_page = 2; 84 | // startCursor is the cursor to the first node in edges. Or the cursor of the representation of the first returned element. 85 | optional string start_cursor = 3 [(grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = {example: "\"aGVsbG8=\""}]; 86 | // endCursor is the cursor to the last node in edges. Or the cursor of the representation of the last returned element. 87 | optional string end_cursor = 4 [(grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = {example: "\"aGVsbG8=\""}]; 88 | } 89 | 90 | message PaginationRequest { 91 | // first is the number of items to return from the beginning of the list. 92 | int64 first = 1; 93 | // after is the cursor to the first node in edges that should be returned. 94 | optional string after = 2 [(grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = {example: "\"aGVsbG8=\""}]; 95 | // last is the number of items to return from the end of the list. 96 | int64 last = 3; 97 | // before is the cursor to the last node in edges that should be returned. 98 | optional string before = 4 [(grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = {example: "\"aGVsbG8=\""}]; 99 | } 100 | -------------------------------------------------------------------------------- /apis/llmgapi/v1/openai/service_grpc.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go-grpc. DO NOT EDIT. 2 | // versions: 3 | // - protoc-gen-go-grpc v1.3.0 4 | // - protoc (unknown) 5 | // source: apis/llmgapi/v1/openai/service.proto 6 | 7 | package openai 8 | 9 | import ( 10 | context "context" 11 | grpc "google.golang.org/grpc" 12 | codes "google.golang.org/grpc/codes" 13 | status "google.golang.org/grpc/status" 14 | ) 15 | 16 | // This is a compile-time assertion to ensure that this generated file 17 | // is compatible with the grpc package it is being compiled against. 18 | // Requires gRPC-Go v1.32.0 or later. 19 | const _ = grpc.SupportPackageIsVersion7 20 | 21 | const ( 22 | OpenAIService_CreateChatCompletion_FullMethodName = "/apis.llmgapi.v1.openai.OpenAIService/CreateChatCompletion" 23 | OpenAIService_CreateChatCompletionStream_FullMethodName = "/apis.llmgapi.v1.openai.OpenAIService/CreateChatCompletionStream" 24 | ) 25 | 26 | // OpenAIServiceClient is the client API for OpenAIService service. 27 | // 28 | // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. 29 | type OpenAIServiceClient interface { 30 | // CreateChatCompletion generates a model response for the given chat conversation. 31 | CreateChatCompletion(ctx context.Context, in *CreateChatCompletionRequest, opts ...grpc.CallOption) (*CreateChatCompletionResponse, error) 32 | // CreateChatCompletionStream generates a streaming model response for the given chat conversation. 33 | // It returns a stream of ChatCompletionChunk messages. 34 | CreateChatCompletionStream(ctx context.Context, in *CreateChatCompletionStreamRequest, opts ...grpc.CallOption) (OpenAIService_CreateChatCompletionStreamClient, error) 35 | } 36 | 37 | type openAIServiceClient struct { 38 | cc grpc.ClientConnInterface 39 | } 40 | 41 | func NewOpenAIServiceClient(cc grpc.ClientConnInterface) OpenAIServiceClient { 42 | return &openAIServiceClient{cc} 43 | } 44 | 45 | func (c *openAIServiceClient) CreateChatCompletion(ctx context.Context, in *CreateChatCompletionRequest, opts ...grpc.CallOption) (*CreateChatCompletionResponse, error) { 46 | out := new(CreateChatCompletionResponse) 47 | err := c.cc.Invoke(ctx, OpenAIService_CreateChatCompletion_FullMethodName, in, out, opts...) 48 | if err != nil { 49 | return nil, err 50 | } 51 | return out, nil 52 | } 53 | 54 | func (c *openAIServiceClient) CreateChatCompletionStream(ctx context.Context, in *CreateChatCompletionStreamRequest, opts ...grpc.CallOption) (OpenAIService_CreateChatCompletionStreamClient, error) { 55 | stream, err := c.cc.NewStream(ctx, &OpenAIService_ServiceDesc.Streams[0], OpenAIService_CreateChatCompletionStream_FullMethodName, opts...) 56 | if err != nil { 57 | return nil, err 58 | } 59 | x := &openAIServiceCreateChatCompletionStreamClient{stream} 60 | if err := x.ClientStream.SendMsg(in); err != nil { 61 | return nil, err 62 | } 63 | if err := x.ClientStream.CloseSend(); err != nil { 64 | return nil, err 65 | } 66 | return x, nil 67 | } 68 | 69 | type OpenAIService_CreateChatCompletionStreamClient interface { 70 | Recv() (*CreateChatCompletionStreamResponse, error) 71 | grpc.ClientStream 72 | } 73 | 74 | type openAIServiceCreateChatCompletionStreamClient struct { 75 | grpc.ClientStream 76 | } 77 | 78 | func (x *openAIServiceCreateChatCompletionStreamClient) Recv() (*CreateChatCompletionStreamResponse, error) { 79 | m := new(CreateChatCompletionStreamResponse) 80 | if err := x.ClientStream.RecvMsg(m); err != nil { 81 | return nil, err 82 | } 83 | return m, nil 84 | } 85 | 86 | // OpenAIServiceServer is the server API for OpenAIService service. 87 | // All implementations must embed UnimplementedOpenAIServiceServer 88 | // for forward compatibility 89 | type OpenAIServiceServer interface { 90 | // CreateChatCompletion generates a model response for the given chat conversation. 91 | CreateChatCompletion(context.Context, *CreateChatCompletionRequest) (*CreateChatCompletionResponse, error) 92 | // CreateChatCompletionStream generates a streaming model response for the given chat conversation. 93 | // It returns a stream of ChatCompletionChunk messages. 94 | CreateChatCompletionStream(*CreateChatCompletionStreamRequest, OpenAIService_CreateChatCompletionStreamServer) error 95 | mustEmbedUnimplementedOpenAIServiceServer() 96 | } 97 | 98 | // UnimplementedOpenAIServiceServer must be embedded to have forward compatible implementations. 99 | type UnimplementedOpenAIServiceServer struct { 100 | } 101 | 102 | func (UnimplementedOpenAIServiceServer) CreateChatCompletion(context.Context, *CreateChatCompletionRequest) (*CreateChatCompletionResponse, error) { 103 | return nil, status.Errorf(codes.Unimplemented, "method CreateChatCompletion not implemented") 104 | } 105 | func (UnimplementedOpenAIServiceServer) CreateChatCompletionStream(*CreateChatCompletionStreamRequest, OpenAIService_CreateChatCompletionStreamServer) error { 106 | return status.Errorf(codes.Unimplemented, "method CreateChatCompletionStream not implemented") 107 | } 108 | func (UnimplementedOpenAIServiceServer) mustEmbedUnimplementedOpenAIServiceServer() {} 109 | 110 | // UnsafeOpenAIServiceServer may be embedded to opt out of forward compatibility for this service. 111 | // Use of this interface is not recommended, as added methods to OpenAIServiceServer will 112 | // result in compilation errors. 113 | type UnsafeOpenAIServiceServer interface { 114 | mustEmbedUnimplementedOpenAIServiceServer() 115 | } 116 | 117 | func RegisterOpenAIServiceServer(s grpc.ServiceRegistrar, srv OpenAIServiceServer) { 118 | s.RegisterService(&OpenAIService_ServiceDesc, srv) 119 | } 120 | 121 | func _OpenAIService_CreateChatCompletion_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { 122 | in := new(CreateChatCompletionRequest) 123 | if err := dec(in); err != nil { 124 | return nil, err 125 | } 126 | if interceptor == nil { 127 | return srv.(OpenAIServiceServer).CreateChatCompletion(ctx, in) 128 | } 129 | info := &grpc.UnaryServerInfo{ 130 | Server: srv, 131 | FullMethod: OpenAIService_CreateChatCompletion_FullMethodName, 132 | } 133 | handler := func(ctx context.Context, req interface{}) (interface{}, error) { 134 | return srv.(OpenAIServiceServer).CreateChatCompletion(ctx, req.(*CreateChatCompletionRequest)) 135 | } 136 | return interceptor(ctx, in, info, handler) 137 | } 138 | 139 | func _OpenAIService_CreateChatCompletionStream_Handler(srv interface{}, stream grpc.ServerStream) error { 140 | m := new(CreateChatCompletionStreamRequest) 141 | if err := stream.RecvMsg(m); err != nil { 142 | return err 143 | } 144 | return srv.(OpenAIServiceServer).CreateChatCompletionStream(m, &openAIServiceCreateChatCompletionStreamServer{stream}) 145 | } 146 | 147 | type OpenAIService_CreateChatCompletionStreamServer interface { 148 | Send(*CreateChatCompletionStreamResponse) error 149 | grpc.ServerStream 150 | } 151 | 152 | type openAIServiceCreateChatCompletionStreamServer struct { 153 | grpc.ServerStream 154 | } 155 | 156 | func (x *openAIServiceCreateChatCompletionStreamServer) Send(m *CreateChatCompletionStreamResponse) error { 157 | return x.ServerStream.SendMsg(m) 158 | } 159 | 160 | // OpenAIService_ServiceDesc is the grpc.ServiceDesc for OpenAIService service. 161 | // It's only intended for direct use with grpc.RegisterService, 162 | // and not to be introspected or modified (even as a copy) 163 | var OpenAIService_ServiceDesc = grpc.ServiceDesc{ 164 | ServiceName: "apis.llmgapi.v1.openai.OpenAIService", 165 | HandlerType: (*OpenAIServiceServer)(nil), 166 | Methods: []grpc.MethodDesc{ 167 | { 168 | MethodName: "CreateChatCompletion", 169 | Handler: _OpenAIService_CreateChatCompletion_Handler, 170 | }, 171 | }, 172 | Streams: []grpc.StreamDesc{ 173 | { 174 | StreamName: "CreateChatCompletionStream", 175 | Handler: _OpenAIService_CreateChatCompletionStream_Handler, 176 | ServerStreams: true, 177 | }, 178 | }, 179 | Metadata: "apis/llmgapi/v1/openai/service.proto", 180 | } 181 | -------------------------------------------------------------------------------- /buf.gen.yaml: -------------------------------------------------------------------------------- 1 | version: v2 2 | plugins: 3 | - remote: buf.build/grpc/go:v1.3.0 4 | out: . 5 | opt: paths=source_relative 6 | 7 | - remote: buf.build/protocolbuffers/go:v1.34.0 8 | out: . 9 | opt: paths=source_relative 10 | 11 | # - remote: buf.build/grpc-ecosystem/gateway:v2.19.1 12 | # out: . 13 | # opt: 14 | # - paths=source_relative 15 | # - allow_delete_body=true 16 | 17 | # - remote: buf.build/grpc-ecosystem/openapiv2:v2.19.1 18 | # out: ./apis/gatewayapi/v1 19 | # opt: 20 | # - file=./apis/gatewayapi/v1 21 | # - merge_file_name=v1 22 | # - allow_merge=true 23 | # - allow_delete_body=true 24 | # - disable_default_errors=true 25 | # - disable_service_tags=true 26 | # - output_format=yaml 27 | 28 | # - name: grpc-gateway-ts 29 | # out: ./clients/typescript 30 | -------------------------------------------------------------------------------- /buf.lock: -------------------------------------------------------------------------------- 1 | # Generated by buf. DO NOT EDIT. 2 | version: v2 3 | deps: 4 | - name: buf.build/bufbuild/protovalidate 5 | commit: a6c49f84cc0f4e038680d390392e2ab0 6 | digest: b5:e968392e88ff7915adcbd1635d670b45bff8836ec2415d81fc559ca5470a695dbdc30030bad8bc5764647c731079e9e7bba0023ea25c4e4a1672a7d2561d4a19 7 | - name: buf.build/googleapis/googleapis 8 | commit: 8bc2c51e08c447cd8886cdea48a73e14 9 | digest: b5:b7e0ac9d192bd0eae88160101269550281448c51f25121cd0d51957661a350aab07001bc145fe9029a8da10b99ff000ae5b284ecaca9c75f2a99604a04d9b4ab 10 | - name: buf.build/grpc-ecosystem/grpc-gateway 11 | commit: a48fcebcf8f140dd9d09359b9bb185a4 12 | digest: b5:330af8a71b579ab96c4f3ee26929d1a68a5a9e986c7cfe0a898591fc514216bb6e723dc04c74d90fdee3f3f14f9100a54b4f079eb273e6e7213f0d5baca36ff8 13 | - name: buf.build/protocolbuffers/wellknowntypes 14 | commit: ee20af7d5b6044139bb9051283720274 15 | digest: b5:690b89ac58ca2ddc9b34eac662d8348754b464c9acc689caf4a4bea613f0704124636a53e5dbc0cc4ea2efeb696b560537f96bc1188419b94c1dcf86c997f6a3 16 | -------------------------------------------------------------------------------- /buf.yaml: -------------------------------------------------------------------------------- 1 | version: v2 2 | breaking: 3 | use: 4 | - FILE 5 | lint: 6 | use: 7 | - DEFAULT 8 | except: 9 | - PACKAGE_VERSION_SUFFIX 10 | - SERVICE_SUFFIX 11 | - ENUM_VALUE_UPPER_SNAKE_CASE 12 | - ENUM_ZERO_VALUE_SUFFIX 13 | - ENUM_VALUE_PREFIX 14 | deps: 15 | - buf.build/googleapis/googleapis 16 | - buf.build/protocolbuffers/wellknowntypes 17 | - buf.build/grpc-ecosystem/grpc-gateway 18 | - buf.build/bufbuild/protovalidate 19 | -------------------------------------------------------------------------------- /cmd/lingticio/llmg-graphql/Dockerfile: -------------------------------------------------------------------------------- 1 | # --- builder --- 2 | FROM golang:1.23 as builder 3 | 4 | ARG BUILD_VERSION 5 | ARG BUILD_LAST_COMMIT 6 | 7 | RUN mkdir /app 8 | RUN mkdir /app/llmg 9 | 10 | WORKDIR /app/llmg 11 | 12 | COPY go.mod /app/llmg/go.mod 13 | COPY go.sum /app/llmg/go.sum 14 | 15 | RUN go env 16 | RUN go env -w CGO_ENABLED=0 17 | RUN go mod download 18 | 19 | COPY . /app/llmg 20 | 21 | RUN go build \ 22 | -a \ 23 | -o "release/lingticio/llmg-graphql" \ 24 | -ldflags " -X './internal/meta.Version=$BUILD_VERSION' -X './internal/meta.LastCommit=$BUILD_LAST_COMMIT'" \ 25 | "./cmd/lingticio/llmg-graphql" 26 | 27 | # --- runner --- 28 | FROM debian as runner 29 | 30 | RUN apt update && apt upgrade -y && apt install -y ca-certificates curl && update-ca-certificates 31 | 32 | COPY --from=builder /app/llmg/release/lingticio/llmg-graphql /app/llmg/release/lingticio/llmg-graphql 33 | 34 | RUN mkdir -p /usr/local/bin/lingticio 35 | RUN ln -s /app/llmg/release/lingticio/llmg-graphql /usr/local/bin/lingticio/llmg-graphql 36 | 37 | ENV LOG_FILE_PATH /var/log/llmg-services/lingticio/llmg-graphql.log 38 | 39 | CMD [ "/usr/local/bin/lingticio/llmg-graphql" ] 40 | -------------------------------------------------------------------------------- /cmd/lingticio/llmg-graphql/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "time" 7 | 8 | "go.uber.org/fx" 9 | 10 | "github.com/lingticio/llmg/internal/configs" 11 | "github.com/lingticio/llmg/internal/datastore" 12 | "github.com/lingticio/llmg/internal/graph/server" 13 | "github.com/lingticio/llmg/internal/libs" 14 | "github.com/spf13/cobra" 15 | ) 16 | 17 | var ( 18 | configFilePath string 19 | envFilePath string 20 | ) 21 | 22 | func main() { 23 | root := &cobra.Command{ 24 | Use: "llmg-graphql", 25 | RunE: func(cmd *cobra.Command, args []string) error { 26 | app := fx.New( 27 | fx.Provide(configs.NewConfig("lingticio", "llmg", configFilePath, envFilePath)), 28 | fx.Options(libs.Modules()), 29 | fx.Options(datastore.Modules()), 30 | fx.Options(server.Modules()), 31 | fx.Invoke(server.Run()), 32 | ) 33 | 34 | app.Run() 35 | 36 | stopCtx, stopCtxCancel := context.WithTimeout(context.Background(), time.Minute*5) 37 | defer stopCtxCancel() 38 | 39 | if err := app.Stop(stopCtx); err != nil { 40 | return err 41 | } 42 | 43 | return nil 44 | }, 45 | } 46 | 47 | root.Flags().StringVarP(&configFilePath, "config", "c", "", "config file path") 48 | root.Flags().StringVarP(&envFilePath, "env", "e", "", "env file path") 49 | 50 | if err := root.Execute(); err != nil { 51 | log.Fatal(err) 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /cmd/lingticio/llmg-grpc/Dockerfile: -------------------------------------------------------------------------------- 1 | # --- builder --- 2 | FROM golang:1.23 as builder 3 | 4 | ARG BUILD_VERSION 5 | ARG BUILD_LAST_COMMIT 6 | 7 | RUN mkdir /app 8 | RUN mkdir /app/llmg 9 | 10 | WORKDIR /app/llmg 11 | 12 | COPY go.mod /app/llmg/go.mod 13 | COPY go.sum /app/llmg/go.sum 14 | 15 | RUN go env 16 | RUN go env -w CGO_ENABLED=0 17 | RUN go mod download 18 | 19 | COPY . /app/llmg 20 | 21 | RUN go build \ 22 | -a \ 23 | -o "release/lingticio/llmg-grpc" \ 24 | -ldflags " -X './internal/meta.Version=$BUILD_VERSION' -X './internal/meta.LastCommit=$BUILD_LAST_COMMIT'" \ 25 | "./cmd/lingticio/llmg-grpc" 26 | 27 | # --- runner --- 28 | FROM debian as runner 29 | 30 | RUN apt update && apt upgrade -y && apt install -y ca-certificates curl && update-ca-certificates 31 | 32 | COPY --from=builder /app/llmg/release/lingticio/llmg-grpc /app/llmg/release/lingticio/llmg-grpc 33 | 34 | RUN mkdir -p /usr/local/bin/lingticio 35 | RUN ln -s /app/llmg/release/lingticio/llmg-grpc /usr/local/bin/lingticio/llmg-grpc 36 | 37 | ENV LOG_FILE_PATH /var/log/llmg-services/lingticio/llmg-grpc.log 38 | 39 | CMD [ "/usr/local/bin/lingticio/llmg-grpc" ] 40 | -------------------------------------------------------------------------------- /cmd/lingticio/llmg-grpc/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "time" 7 | 8 | "go.uber.org/fx" 9 | 10 | "github.com/lingticio/llmg/internal/configs" 11 | "github.com/lingticio/llmg/internal/datastore" 12 | grpcservers "github.com/lingticio/llmg/internal/grpc/servers" 13 | v1 "github.com/lingticio/llmg/internal/grpc/servers/llmg/v1" 14 | grpcservices "github.com/lingticio/llmg/internal/grpc/services" 15 | "github.com/lingticio/llmg/internal/libs" 16 | "github.com/spf13/cobra" 17 | ) 18 | 19 | var ( 20 | configFilePath string 21 | envFilePath string 22 | ) 23 | 24 | func main() { 25 | root := &cobra.Command{ 26 | Use: "llmg-grpc", 27 | RunE: func(cmd *cobra.Command, args []string) error { 28 | app := fx.New( 29 | fx.Provide(configs.NewConfig("lingticio", "llmg", configFilePath, envFilePath)), 30 | fx.Options(libs.Modules()), 31 | fx.Options(datastore.Modules()), 32 | fx.Options(grpcservers.Modules()), 33 | fx.Options(grpcservices.Modules()), 34 | fx.Invoke(v1.Run()), 35 | ) 36 | 37 | app.Run() 38 | 39 | stopCtx, stopCtxCancel := context.WithTimeout(context.Background(), time.Minute*5) 40 | defer stopCtxCancel() 41 | 42 | if err := app.Stop(stopCtx); err != nil { 43 | return err 44 | } 45 | 46 | return nil 47 | }, 48 | } 49 | 50 | root.Flags().StringVarP(&configFilePath, "config", "c", "", "config file path") 51 | root.Flags().StringVarP(&envFilePath, "env", "e", "", "env file path") 52 | 53 | if err := root.Execute(); err != nil { 54 | log.Fatal(err) 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /cmd/tools/openapiv2conv/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "os" 8 | "strings" 9 | 10 | "github.com/getkin/kin-openapi/openapi2" 11 | "github.com/getkin/kin-openapi/openapi2conv" 12 | "github.com/nekomeowww/xo" 13 | "github.com/spf13/cobra" 14 | "gopkg.in/yaml.v3" 15 | ) 16 | 17 | var ( 18 | input string 19 | output string 20 | ) 21 | 22 | func main() { 23 | root := &cobra.Command{ 24 | Use: "openapiv2conv", 25 | RunE: func(cmd *cobra.Command, args []string) error { 26 | if !strings.HasPrefix(input, "/") { 27 | input = xo.RelativePathBasedOnPwdOf(input) 28 | } 29 | 30 | openapiV2DocContent, err := os.ReadFile(input) 31 | if err != nil { 32 | return fmt.Errorf("failed to read input file %s: %w", input, err) 33 | } 34 | 35 | var openapiV2Doc openapi2.T 36 | 37 | err = json.Unmarshal(openapiV2DocContent, &openapiV2Doc) 38 | if err != nil { 39 | return fmt.Errorf("failed to unmarshal input file %s: %w", input, err) 40 | } 41 | 42 | openapiV3Doc, err := openapi2conv.ToV3(&openapiV2Doc) 43 | if err != nil { 44 | return fmt.Errorf("failed to convert openapi v2 to v3: %w", err) 45 | } 46 | 47 | openapiV3DocBuffer := new(bytes.Buffer) 48 | encoder := yaml.NewEncoder(openapiV3DocBuffer) 49 | encoder.SetIndent(2) //nolint:mnd 50 | 51 | err = encoder.Encode(openapiV3Doc) 52 | if err != nil { 53 | return fmt.Errorf("failed to encode openapi v3 doc: %w", err) 54 | } 55 | 56 | if !strings.HasPrefix(output, "/") { 57 | output = xo.RelativePathBasedOnPwdOf(output) 58 | } 59 | 60 | err = os.WriteFile(output, openapiV3DocBuffer.Bytes(), 0644) //nolint 61 | if err != nil { 62 | return fmt.Errorf("failed to write output file %s: %w", output, err) 63 | } 64 | 65 | return nil 66 | }, 67 | } 68 | 69 | root.Flags().StringVarP(&input, "input", "i", "", "input file path") 70 | root.Flags().StringVarP(&output, "output", "o", "", "output file path") 71 | 72 | err := root.Execute() 73 | if err != nil { 74 | panic(err) 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /config/config.example.yaml: -------------------------------------------------------------------------------- 1 | env: local 2 | 3 | http: 4 | server_addr: :8080 5 | grpc: 6 | server_addr: :8081 7 | -------------------------------------------------------------------------------- /cspell.config.yaml: -------------------------------------------------------------------------------- 1 | version: "0.2" 2 | ignorePaths: 3 | - go.sum 4 | - go.mod 5 | dictionaryDefinitions: [] 6 | dictionaries: [] 7 | words: 8 | - apierrors 9 | - apiserver 10 | - APIURL 11 | - Balancable 12 | - bufbuild 13 | - containedctx 14 | - contextcheck 15 | - coreapi 16 | - coreapiv1 17 | - cyclop 18 | - depguard 19 | - Detailf 20 | - dupl 21 | - entgo 22 | - errorlint 23 | - execinquery 24 | - exhaustruct 25 | - forbidigo 26 | - funlen 27 | - gatewayapi 28 | - gatewayapiv 29 | - getkin 30 | - gochecknoglobals 31 | - gocognit 32 | - gocritic 33 | - gocyclo 34 | - godotenv 35 | - godox 36 | - gofumpt 37 | - gomnd 38 | - gonanoid 39 | - googleapis 40 | - gosec 41 | - gqlerror 42 | - gqlgen 43 | - grpcpkg 44 | - grpcservers 45 | - grpcservices 46 | - ireturn 47 | - jsonapi 48 | - jsonfmt 49 | - jsonschema 50 | - jwtclaim 51 | - labstack 52 | - lingtic 53 | - lingticio 54 | - llmg 55 | - llmgapi 56 | - Logit 57 | - Logprob 58 | - maintidx 59 | - managementapi 60 | - managementapiv1 61 | - mapstructure 62 | - matoous 63 | - mitchellh 64 | - modelv1 65 | - Nargs 66 | - nekomeowww 67 | - nestif 68 | - Nillable 69 | - nilnil 70 | - nlreturn 71 | - nolint 72 | - nonamedreturns 73 | - openai 74 | - openaiapiv1 75 | - openapi 76 | - openapiv2 77 | - openapiv3 78 | - paralleltest 79 | - perfsprint 80 | - Probs 81 | - Protobuf 82 | - protoc_gen_openapiv2 83 | - protoc-gen-openapiv2 84 | - protocolbuffers 85 | - protoiface 86 | - protovalidate 87 | - rediskeys 88 | - Rueidis 89 | - rueidisstore 90 | - samber 91 | - santhosh-tekuri 92 | - sashabaranov 93 | - semanticcache 94 | - serverv1 95 | - Sortby 96 | - stretchr 97 | - stylecheck 98 | - Supabase 99 | - tagalign 100 | - testpackage 101 | - thumbhash 102 | - timestamppb 103 | - Upgrader 104 | - Upstreamable 105 | - Upstreams 106 | - varnamelen 107 | - vektah 108 | - wellknowntypes 109 | - wrapcheck 110 | ignoreWords: [] 111 | import: 112 | - "@cspell/dict-golang" 113 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.9" 2 | 3 | services: 4 | redis_stack_local: 5 | image: redis/redis-stack-server:latest 6 | restart: unless-stopped 7 | ports: 8 | - 6379:6379 9 | 10 | # # promtail service that helps services to collect logs 11 | # # you can uncomment the following lines if you want to enable promtail 12 | # promtail: 13 | # image: grafana/promtail:2.8.2 14 | # restart: unless-stopped 15 | # volumes: 16 | # - ./production/promtail:/etc/promtail # you may edit the ./production/promtail/config.yaml file to suit your needs 17 | # - logs/node-api-server:/var/log/services/node-api-server:ro 18 | # command: > 19 | # -config.file=/etc/promtail/config.yaml 20 | 21 | # # uncomment the following lines if you want to view the web ui of promtail 22 | # # ports: 23 | # # - 9080:9080 24 | # # uncomment the following lines if you want to use your own loki service 25 | # # networks: 26 | # # - # replace with your own loki network name 27 | # 28 | # uncomment the following lines if you want to use your own loki service 29 | # that live in another docker-compose.yml file 30 | # networks: 31 | # : # replace with your own loki network name 32 | # external: true 33 | -------------------------------------------------------------------------------- /docs/designs/1 - Configs.md: -------------------------------------------------------------------------------- 1 | # Configs 2 | 3 | In order to make configurations reloadable, part of the configs multi-sourcable, and part of the configs environment-specific, we need to have a clear structure for our configs. 4 | 5 | ## Data Sources 6 | 7 | There are multiple ways of reading configs for Gateway: 8 | 9 | - Redis 10 | - Plain configuration files 11 | - Postgres, MySQL, etc. 12 | 13 | To support diverged data sources, configurations must divide into two parts: 14 | 15 | - **Static Configs**: These are the configurations that are read from the configuration files. They are static and do not change during the runtime. 16 | - **Dynamic Configs**: These are the configurations that are read from the Redis. They are dynamic and can be changed during the runtime. 17 | 18 | For the static configs, 19 | 20 | - Redis Connection Strings 21 | - Redis Keys 22 | - DB Connection Strings (Postgres, MySQL) 23 | 24 | ## Hierarchy, Groups, Items 25 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/lingticio/llmg 2 | 3 | go 1.23.0 4 | 5 | toolchain go1.23.2 6 | 7 | require ( 8 | github.com/99designs/gqlgen v0.17.61 9 | github.com/bufbuild/protovalidate-go v0.8.0 10 | github.com/getkin/kin-openapi v0.128.0 11 | github.com/golang-jwt/jwt v3.2.2+incompatible 12 | github.com/gorilla/websocket v1.5.3 13 | github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1 14 | github.com/joho/godotenv v1.5.1 15 | github.com/labstack/echo/v4 v4.13.3 16 | github.com/matoous/go-nanoid/v2 v2.1.0 17 | github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1 18 | github.com/mitchellh/mapstructure v1.5.0 19 | github.com/nekomeowww/fo v1.4.0 20 | github.com/nekomeowww/xo v1.14.0 21 | github.com/patrickmn/go-cache v2.1.0+incompatible 22 | github.com/redis/rueidis v1.0.51 23 | github.com/redis/rueidis/om v1.0.51 24 | github.com/rivo/uniseg v0.4.7 25 | github.com/samber/lo v1.47.0 26 | github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 27 | github.com/sashabaranov/go-openai v1.36.0 28 | github.com/spf13/cobra v1.8.1 29 | github.com/spf13/viper v1.19.0 30 | github.com/stretchr/testify v1.10.0 31 | github.com/vektah/gqlparser/v2 v2.5.20 32 | go.uber.org/fx v1.23.0 33 | go.uber.org/zap v1.27.0 34 | google.golang.org/grpc v1.69.2 35 | google.golang.org/protobuf v1.36.0 36 | gopkg.in/yaml.v3 v3.0.1 37 | ) 38 | 39 | require ( 40 | buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.0-20241127180247-a33202765966.1 // indirect 41 | cel.dev/expr v0.19.1 // indirect 42 | entgo.io/ent v0.14.1 // indirect 43 | github.com/agnivade/levenshtein v1.2.0 // indirect 44 | github.com/antlr4-go/antlr/v4 v4.13.1 // indirect 45 | github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect 46 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 47 | github.com/fsnotify/fsnotify v1.8.0 // indirect 48 | github.com/go-logr/logr v1.4.2 // indirect 49 | github.com/go-logr/stdr v1.2.2 // indirect 50 | github.com/go-openapi/jsonpointer v0.21.0 // indirect 51 | github.com/go-openapi/swag v0.23.0 // indirect 52 | github.com/go-viper/mapstructure/v2 v2.2.1 // indirect 53 | github.com/google/cel-go v0.22.1 // indirect 54 | github.com/google/uuid v1.6.0 // indirect 55 | github.com/gookit/color v1.5.4 // indirect 56 | github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect 57 | github.com/hashicorp/hcl v1.0.0 // indirect 58 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 59 | github.com/invopop/yaml v0.3.1 // indirect 60 | github.com/josharian/intern v1.0.0 // indirect 61 | github.com/labstack/gommon v0.4.2 // indirect 62 | github.com/magiconair/properties v1.8.9 // indirect 63 | github.com/mailru/easyjson v0.9.0 // indirect 64 | github.com/mattn/go-colorable v0.1.13 // indirect 65 | github.com/mattn/go-isatty v0.0.20 // indirect 66 | github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect 67 | github.com/oklog/ulid/v2 v2.1.0 // indirect 68 | github.com/pelletier/go-toml/v2 v2.2.3 // indirect 69 | github.com/perimeterx/marshmallow v1.1.5 // indirect 70 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 71 | github.com/russross/blackfriday/v2 v2.1.0 // indirect 72 | github.com/sagikazarmark/locafero v0.6.0 // indirect 73 | github.com/sagikazarmark/slog-shim v0.1.0 // indirect 74 | github.com/shopspring/decimal v1.4.0 // indirect 75 | github.com/sirupsen/logrus v1.9.3 // indirect 76 | github.com/sosodev/duration v1.3.1 // indirect 77 | github.com/sourcegraph/conc v0.3.0 // indirect 78 | github.com/spf13/afero v1.11.0 // indirect 79 | github.com/spf13/cast v1.7.1 // indirect 80 | github.com/spf13/pflag v1.0.5 // indirect 81 | github.com/stoewer/go-strcase v1.3.0 // indirect 82 | github.com/subosito/gotenv v1.6.0 // indirect 83 | github.com/urfave/cli/v2 v2.27.5 // indirect 84 | github.com/valyala/bytebufferpool v1.0.0 // indirect 85 | github.com/valyala/fasttemplate v1.2.2 // indirect 86 | github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect 87 | github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect 88 | go.opentelemetry.io/auto/sdk v1.1.0 // indirect 89 | go.opentelemetry.io/otel v1.33.0 // indirect 90 | go.opentelemetry.io/otel/log v0.9.0 // indirect 91 | go.opentelemetry.io/otel/metric v1.33.0 // indirect 92 | go.opentelemetry.io/otel/trace v1.33.0 // indirect 93 | go.uber.org/dig v1.18.0 // indirect 94 | go.uber.org/multierr v1.11.0 // indirect 95 | golang.org/x/crypto v0.31.0 // indirect 96 | golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67 // indirect 97 | golang.org/x/mod v0.22.0 // indirect 98 | golang.org/x/net v0.33.0 // indirect 99 | golang.org/x/sync v0.10.0 // indirect 100 | golang.org/x/sys v0.28.0 // indirect 101 | golang.org/x/text v0.21.0 // indirect 102 | golang.org/x/time v0.8.0 // indirect 103 | golang.org/x/tools v0.28.0 // indirect 104 | google.golang.org/genproto/googleapis/api v0.0.0-20241219192143-6b3ec007d9bb // indirect 105 | google.golang.org/genproto/googleapis/rpc v0.0.0-20241219192143-6b3ec007d9bb // indirect 106 | gopkg.in/ini.v1 v1.67.0 // indirect 107 | ) 108 | -------------------------------------------------------------------------------- /graph/openai/gqlgen.yml: -------------------------------------------------------------------------------- 1 | # Where are all the schema files located? globs are supported eg src/**/*.graphqls 2 | schema: 3 | - ../../../graph/openai/*.graphqls 4 | 5 | # Where should the generated server code go? 6 | exec: 7 | filename: generated/generated.go 8 | package: generated 9 | 10 | # Uncomment to enable federation 11 | # federation: 12 | # filename: graph/generated/federation.go 13 | # package: generated 14 | 15 | # Where should any generated models go? 16 | model: 17 | filename: model/models_gen.go 18 | package: model 19 | 20 | # Where should the resolver implementations go? 21 | resolver: 22 | layout: follow-schema 23 | dir: resolvers 24 | package: resolvers 25 | filename_template: "{name}.resolver.go" 26 | # Optional: turn on to not generate template comments above resolvers 27 | # omit_template_comment: false 28 | 29 | # Optional: turn on use ` + "`" + `gqlgen:"fieldName"` + "`" + ` tags in your models 30 | # struct_tag: json 31 | 32 | # Optional: turn on to use []Thing instead of []*Thing 33 | # omit_slice_element_pointers: false 34 | 35 | # Optional: turn on to omit Is() methods to interface and unions 36 | # omit_interface_checks : true 37 | 38 | # Optional: turn on to skip generation of ComplexityRoot struct content and Complexity function 39 | # omit_complexity: false 40 | 41 | # Optional: turn on to not generate any file notice comments in generated files 42 | # omit_gqlgen_file_notice: false 43 | 44 | # Optional: turn on to exclude the gqlgen version in the generated file notice. No effect if `omit_gqlgen_file_notice` is true. 45 | # omit_gqlgen_version_in_file_notice: false 46 | 47 | # Optional: turn off to make struct-type struct fields not use pointers 48 | # e.g. type Thing struct { FieldA OtherThing } instead of { FieldA *OtherThing } 49 | # struct_fields_always_pointers: true 50 | 51 | # Optional: turn off to make resolvers return values instead of pointers for structs 52 | # resolvers_always_return_pointers: true 53 | 54 | # Optional: turn on to return pointers instead of values in unmarshalInput 55 | # return_pointers_in_unmarshalinput: false 56 | 57 | # Optional: wrap nullable input fields with Omittable 58 | # nullable_input_omittable: true 59 | 60 | # Optional: set to speed up generation time by not performing a final validation pass. 61 | # skip_validation: true 62 | 63 | # Optional: set to skip running `go mod tidy` when generating server code 64 | # skip_mod_tidy: true 65 | 66 | # gqlgen will search for any type names in the schema in these go packages 67 | # if they match it will use them, otherwise it will generate them. 68 | autobind: 69 | # - "github.com/lingticio/llmg/graph/model" 70 | 71 | # This section declares type mapping between the GraphQL and go type systems 72 | # 73 | # The first line in each type will be used as defaults for resolver arguments and 74 | # modelgen, the others will be allowed when binding to fields. Configure them to 75 | # your liking 76 | models: 77 | ID: 78 | model: 79 | - github.com/99designs/gqlgen/graphql.ID 80 | - github.com/99designs/gqlgen/graphql.Int 81 | - github.com/99designs/gqlgen/graphql.Int64 82 | - github.com/99designs/gqlgen/graphql.Int32 83 | Int: 84 | model: 85 | - github.com/99designs/gqlgen/graphql.Int 86 | - github.com/99designs/gqlgen/graphql.Int64 87 | - github.com/99designs/gqlgen/graphql.Int32 88 | -------------------------------------------------------------------------------- /hack/proto-export: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | SCRIPT_PATH=$(realpath "$0") 4 | SCRIPT_DIR=$(dirname "$SCRIPT_PATH") 5 | PROTO_VENDOR_DIR="$SCRIPT_DIR/../.temp/cache/buf.build/vendor/proto" 6 | 7 | buf dep update 8 | yq '.deps' buf.yaml | sed 's|^-*||' | xargs -I {} buf export {} --output $PROTO_VENDOR_DIR 9 | -------------------------------------------------------------------------------- /hack/proto-gen: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | SCRIPT_PATH=$(realpath "$0") 4 | SCRIPT_DIR=$(dirname "$SCRIPT_PATH") 5 | PROTO_APIS_DIR="$SCRIPT_DIR/../apis" 6 | 7 | buf generate --path "$PROTO_APIS_DIR" 8 | 9 | # go build \ 10 | # -a \ 11 | # -o "release/tools/openapiv2conv" \ 12 | # "./cmd/tools/openapiv2conv" 13 | 14 | # chmod +x release/tools/openapiv2conv 15 | 16 | # ./release/tools/openapiv2conv -i apis/coreapi/v1/v1.swagger.json -o apis/coreapi/v1/v1.swagger.v3.yaml 17 | # ./release/tools/openapiv2conv -i apis/managementapi/v1/v1.swagger.json -o apis/managementapi/v1/v1.swagger.v3.yaml 18 | -------------------------------------------------------------------------------- /internal/configs/common.go: -------------------------------------------------------------------------------- 1 | package configs 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | "strings" 9 | 10 | "github.com/joho/godotenv" 11 | "github.com/nekomeowww/xo" 12 | "github.com/spf13/viper" 13 | ) 14 | 15 | const ConfigFilePathEnvName = "CONFIG_FILE_PATH" 16 | 17 | func getConfigFilePath(configFilePath string) string { 18 | if configFilePath != "" { 19 | return configFilePath 20 | } 21 | 22 | envPath := os.Getenv(ConfigFilePathEnvName) 23 | if envPath != "" { 24 | return envPath 25 | } 26 | 27 | configPath := xo.RelativePathBasedOnPwdOf("config/config.yaml") 28 | 29 | return configPath 30 | } 31 | 32 | var ( 33 | possibleConfigPathsForUnitTest = []string{ 34 | "config.local.yml", 35 | "config.local.yaml", 36 | "config.test.yml", 37 | "config.test.yaml", 38 | "config.example.yml", 39 | "config.example.yaml", 40 | } 41 | ) 42 | 43 | func tryToMatchConfigPathForUnitTest(configFilePath string) string { 44 | if getConfigFilePath(configFilePath) != "" { 45 | return configFilePath 46 | } 47 | 48 | for _, path := range possibleConfigPathsForUnitTest { 49 | stat, err := os.Stat(filepath.Join(xo.RelativePathOf("../../config"), path)) 50 | if err == nil { 51 | if stat.IsDir() { 52 | panic("config file path is a directory: " + path) 53 | } 54 | 55 | return path 56 | } 57 | if errors.Is(err, os.ErrNotExist) { 58 | continue 59 | } else { 60 | panic(err) 61 | } 62 | } 63 | 64 | return "" 65 | } 66 | 67 | func loadEnvConfig(path string) error { 68 | err := godotenv.Load(path) 69 | if err != nil { 70 | if errors.Is(err, os.ErrNotExist) { 71 | err := godotenv.Load(xo.RelativePathBasedOnPwdOf("./.env")) 72 | if err != nil && !errors.Is(err, os.ErrNotExist) { 73 | return err 74 | } 75 | 76 | return nil 77 | } 78 | 79 | return err 80 | } 81 | 82 | return nil 83 | } 84 | 85 | func readConfig(path string) error { 86 | viper.SetConfigName("app") 87 | viper.SetConfigType("yaml") 88 | viper.SetConfigFile(path) 89 | 90 | viper.AutomaticEnv() 91 | viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) 92 | 93 | err := viper.ReadInConfig() 94 | if err != nil { 95 | var notFoundErr viper.ConfigFileNotFoundError 96 | if errors.As(err, ¬FoundErr) { 97 | return nil 98 | } 99 | if os.IsNotExist(err) { 100 | return nil 101 | } 102 | 103 | return fmt.Errorf("error occurred when read in config, error is: %T, err: %w", err, err) //nolint:errorlint 104 | } 105 | 106 | return nil 107 | } 108 | -------------------------------------------------------------------------------- /internal/configs/configs.go: -------------------------------------------------------------------------------- 1 | package configs 2 | 3 | import ( 4 | "github.com/lingticio/llmg/internal/meta" 5 | "github.com/lingticio/llmg/pkg/util/utils" 6 | "github.com/mitchellh/mapstructure" 7 | "github.com/spf13/viper" 8 | ) 9 | 10 | func NewConfig(namespace string, app string, configFilePath string, envFilePath string) func() (*Config, error) { 11 | return func() (*Config, error) { 12 | var configPath string 13 | if utils.IsInUnitTest() { 14 | configPath = tryToMatchConfigPathForUnitTest(configFilePath) 15 | } else { 16 | configPath = getConfigFilePath(configFilePath) 17 | } 18 | 19 | registerLingticIoCoreConfig() 20 | 21 | err := loadEnvConfig(envFilePath) 22 | if err != nil { 23 | return nil, err 24 | } 25 | 26 | err = readConfig(configPath) 27 | if err != nil { 28 | return nil, err 29 | } 30 | 31 | config := defaultConfig() 32 | 33 | err = viper.Unmarshal(&config, func(c *mapstructure.DecoderConfig) { 34 | c.TagName = "yaml" 35 | }) 36 | if err != nil { 37 | return nil, err 38 | } 39 | 40 | meta.Env = config.Env 41 | 42 | config.Meta.Env = config.Env 43 | config.Meta.App = app 44 | config.Meta.Namespace = namespace 45 | 46 | return &config, nil 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /internal/configs/llmg.go: -------------------------------------------------------------------------------- 1 | package configs 2 | 3 | import ( 4 | "github.com/lingticio/llmg/internal/meta" 5 | "github.com/lingticio/llmg/pkg/types/metadata" 6 | "github.com/samber/lo" 7 | "github.com/spf13/viper" 8 | ) 9 | 10 | type HTTPServer struct { 11 | Addr string `json:"server_addr" yaml:"server_addr"` 12 | } 13 | 14 | type GrpcServer struct { 15 | Addr string `json:"server_addr" yaml:"server_addr"` 16 | } 17 | 18 | type GraphQLServer struct { 19 | Addr string `json:"server_addr" yaml:"server_addr"` 20 | } 21 | 22 | type Endpoint struct { 23 | ID string `json:"id" yaml:"id"` 24 | Alias string `json:"alias" yaml:"alias"` 25 | APIKey string `json:"api_key" yaml:"api_key"` 26 | Upstream *metadata.UpstreamSingleOrMultiple `json:"upstream,omitempty" yaml:"upstream,omitempty"` 27 | } 28 | 29 | type Group struct { 30 | ID string `json:"id" yaml:"id"` 31 | 32 | Groups []Group `json:"groups" yaml:"groups"` 33 | Endpoints []Endpoint `json:"endpoints" yaml:"endpoints"` 34 | Upstream *metadata.UpstreamSingleOrMultiple `json:"upstream,omitempty" yaml:"upstream,omitempty"` 35 | } 36 | 37 | type Team struct { 38 | ID string `json:"id" yaml:"id"` 39 | 40 | Groups []Group `json:"groups" yaml:"groups"` 41 | Upstream *metadata.UpstreamSingleOrMultiple `json:"upstream,omitempty" yaml:"upstream,omitempty"` 42 | } 43 | 44 | type Tenant struct { 45 | ID string `json:"id" yaml:"id"` 46 | 47 | Teams []Team `json:"teams" yaml:"teams"` 48 | Upstream *metadata.UpstreamSingleOrMultiple `json:"upstream,omitempty" yaml:"upstream,omitempty"` 49 | } 50 | 51 | type Routes struct { 52 | Tenants []Tenant `json:"tenants" yaml:"tenants"` 53 | Upstream *metadata.UpstreamSingleOrMultiple `json:"upstream,omitempty" yaml:"upstream,omitempty"` 54 | } 55 | 56 | type Config struct { 57 | meta.Meta `json:"-" yaml:"-"` 58 | 59 | Env string `json:"env" yaml:"env"` 60 | 61 | HTTP HTTPServer `json:"http" yaml:"http"` 62 | Grpc GrpcServer `json:"grpc" yaml:"grpc"` 63 | GraphQL GraphQLServer `json:"graphql" yaml:"graphql"` 64 | Routes Routes `json:"configs" yaml:"configs"` 65 | } 66 | 67 | func defaultConfig() Config { 68 | return Config{ 69 | HTTP: HTTPServer{ 70 | Addr: ":8080", 71 | }, 72 | Grpc: GrpcServer{ 73 | Addr: ":8081", 74 | }, 75 | GraphQL: GraphQLServer{ 76 | Addr: ":8082", 77 | }, 78 | } 79 | } 80 | 81 | func registerLingticIoCoreConfig() { 82 | lo.Must0(viper.BindEnv("lingticio.llmg.http.server_addr")) 83 | lo.Must0(viper.BindEnv("lingticio.llmg.grpc.server_addr")) 84 | lo.Must0(viper.BindEnv("lingticio.llmg.graphql.server_addr")) 85 | } 86 | -------------------------------------------------------------------------------- /internal/configs/types.go: -------------------------------------------------------------------------------- 1 | package configs 2 | 3 | type S3 struct { 4 | AccessKeyID string `json:"access_key_id" yaml:"access_key_id"` 5 | SecretAccessKey string `json:"secret_access_key" yaml:"secret_access_key"` 6 | BucketName string `json:"bucket_name" yaml:"bucket_name"` 7 | Region string `json:"region" yaml:"region"` 8 | Endpoint string `json:"endpoint" yaml:"endpoint"` 9 | } 10 | 11 | type Redis struct { 12 | Host string `json:"host" yaml:"host"` 13 | Port string `json:"port" yaml:"port"` 14 | TLSEnabled bool `json:"tls_enabled" yaml:"tls_enabled"` 15 | Username string `json:"username" yaml:"username"` 16 | Password string `json:"password" yaml:"password"` 17 | DB int64 `json:"db" yaml:"db"` 18 | ClientCacheEnabled bool `json:"client_cache_enabled" yaml:"client_cache_enabled"` 19 | } 20 | 21 | type Database struct { 22 | ConnectionString string `json:"connection_string" yaml:"connection_string"` 23 | } 24 | -------------------------------------------------------------------------------- /internal/constants/cookie.go: -------------------------------------------------------------------------------- 1 | package constants 2 | 3 | const ( 4 | SupabaseAccessTokenCookieName = "sb-access-token" 5 | ) 6 | -------------------------------------------------------------------------------- /internal/constants/metadata.go: -------------------------------------------------------------------------------- 1 | package constants 2 | 3 | const ( 4 | MetadataUserProfileWithSupabaseKey string = "lingticio-supabase-user-profile" 5 | MetadataUserProfileWithAuthorizationKey string = "lingticio-authorization-user-profile" 6 | MetadataBearerTokenKey string = "lingticio-authorization-bearer-token" //nolint:gosec 7 | ) 8 | -------------------------------------------------------------------------------- /internal/constants/sizes.go: -------------------------------------------------------------------------------- 1 | package constants 2 | 3 | const ( 4 | BytesUnitKB = 1024 5 | BytesUnitMB = 1024 * BytesUnitKB 6 | BytesUnitGB = 1024 * BytesUnitMB 7 | ) 8 | -------------------------------------------------------------------------------- /internal/datastore/cache.go: -------------------------------------------------------------------------------- 1 | package datastore 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/patrickmn/go-cache" 7 | "go.uber.org/fx" 8 | ) 9 | 10 | type NewCacheParams struct { 11 | fx.In 12 | } 13 | 14 | type Cache struct { 15 | *cache.Cache 16 | } 17 | 18 | func NewCache() func(params NewCacheParams) (*Cache, error) { 19 | return func(params NewCacheParams) (*Cache, error) { 20 | return &Cache{ 21 | Cache: cache.New(time.Hour, time.Hour), 22 | }, nil 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /internal/datastore/datastore.go: -------------------------------------------------------------------------------- 1 | package datastore 2 | 3 | import "go.uber.org/fx" 4 | 5 | func Modules() fx.Option { 6 | return fx.Options( 7 | fx.Provide(NewCache()), 8 | fx.Provide(NewRueidis()), 9 | ) 10 | } 11 | -------------------------------------------------------------------------------- /internal/datastore/rueidis.go: -------------------------------------------------------------------------------- 1 | package datastore 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/redis/rueidis" 8 | ) 9 | 10 | func NewRueidis() func() (rueidis.Client, error) { 11 | return func() (rueidis.Client, error) { 12 | client, err := rueidis.NewClient(rueidis.ClientOption{ 13 | InitAddress: []string{"localhost:6379"}, 14 | }) 15 | if err != nil { 16 | return nil, err 17 | } 18 | 19 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 20 | defer cancel() 21 | 22 | cmd := client.B().Ping().Build() 23 | 24 | err = client.Do(ctx, cmd).Error() 25 | if err != nil { 26 | return nil, err 27 | } 28 | 29 | return client, nil 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /internal/graph/openai/openai.go: -------------------------------------------------------------------------------- 1 | package openai 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/99designs/gqlgen/graphql/handler" 7 | "github.com/99designs/gqlgen/graphql/handler/transport" 8 | "github.com/99designs/gqlgen/graphql/playground" 9 | "github.com/gorilla/websocket" 10 | "github.com/labstack/echo/v4" 11 | "github.com/lingticio/llmg/internal/graph/openai/generated" 12 | "github.com/lingticio/llmg/internal/graph/openai/resolvers" 13 | "github.com/nekomeowww/xo/logger" 14 | "go.uber.org/fx" 15 | ) 16 | 17 | //go:generate go run github.com/99designs/gqlgen generate --config ../../../graph/openai/gqlgen.yml 18 | 19 | type NewGraphQLHandlerParams struct { 20 | fx.In 21 | 22 | Logger *logger.Logger 23 | } 24 | 25 | type GraphQLHandler struct { 26 | logger *logger.Logger 27 | } 28 | 29 | func NewGraphQLHandler() func(params NewGraphQLHandlerParams) *GraphQLHandler { 30 | return func(params NewGraphQLHandlerParams) *GraphQLHandler { 31 | return &GraphQLHandler{ 32 | logger: params.Logger, 33 | } 34 | } 35 | } 36 | 37 | func (h *GraphQLHandler) InstallForEcho(endpoint string, e *echo.Echo) { 38 | graphqlHandler := handler.NewDefaultServer(generated.NewExecutableSchema(generated.Config{Resolvers: &resolvers.Resolver{Logger: h.logger}})) 39 | 40 | // As documentation of Subscriptions — gqlgen https://gqlgen.com/recipes/subscriptions/ 41 | // has stated, websocket transport is needed for subscriptions. 42 | // 43 | // But it is possible for future implementation to use other transport types. (e.g. SSE) 44 | graphqlHandler.AddTransport(&transport.Websocket{ 45 | Upgrader: websocket.Upgrader{ 46 | CheckOrigin: func(r *http.Request) bool { 47 | return true 48 | }, 49 | ReadBufferSize: 1024, //nolint:mnd 50 | WriteBufferSize: 1024, //nolint:mnd 51 | }, 52 | }) 53 | 54 | // As documentation of Subscriptions — gqlgen https://gqlgen.com/recipes/subscriptions/ 55 | // has stated, POST only handler will not gonna work for subscriptions. 56 | // Therefore e.Any is used to handle all request methods. 57 | e.Any(endpoint, func(c echo.Context) error { 58 | graphqlHandler.ServeHTTP(c.Response(), c.Request()) 59 | return nil 60 | }) 61 | } 62 | 63 | func (h *GraphQLHandler) InstallPlaygroundForEcho(endpoint string, playgroundEndpoint string, e *echo.Echo) { 64 | playgroundHandler := playground.Handler("GraphQL", endpoint) 65 | 66 | e.GET(playgroundEndpoint, func(c echo.Context) error { 67 | playgroundHandler.ServeHTTP(c.Response(), c.Request()) 68 | return nil 69 | }) 70 | } 71 | -------------------------------------------------------------------------------- /internal/graph/openai/resolvers/chat.resolver.go: -------------------------------------------------------------------------------- 1 | package resolvers 2 | 3 | // This file will be automatically regenerated based on the schema, any resolver implementations 4 | // will be copied through when generating and any unknown code will be moved to the end. 5 | // Code generated by github.com/99designs/gqlgen version v0.17.49 6 | 7 | import ( 8 | "context" 9 | "errors" 10 | "fmt" 11 | "io" 12 | 13 | "github.com/99designs/gqlgen/graphql/handler/transport" 14 | "github.com/lingticio/llmg/internal/graph/openai/generated" 15 | "github.com/lingticio/llmg/internal/graph/openai/model" 16 | "github.com/lingticio/llmg/internal/graph/server/middlewares" 17 | "github.com/samber/lo" 18 | "github.com/sashabaranov/go-openai" 19 | "github.com/vektah/gqlparser/v2/gqlerror" 20 | "go.uber.org/zap" 21 | ) 22 | 23 | // CreateChatCompletion is the resolver for the createChatCompletion field. 24 | func (r *mutationResolver) CreateChatCompletion(ctx context.Context, input model.CreateChatCompletionInput) (*model.ChatCompletionResult, error) { 25 | apiKey := middlewares.APIKeyFromContext(ctx) 26 | if apiKey == "" { 27 | if input.APIKey != nil { 28 | apiKey = *input.APIKey 29 | } 30 | } 31 | 32 | config := openai.DefaultConfig(apiKey) 33 | baseURL := middlewares.XBaseURLFromContext(ctx) 34 | if baseURL != "" { 35 | config.BaseURL = baseURL 36 | } 37 | if input.BaseURL != nil { 38 | config.BaseURL = *input.BaseURL 39 | } 40 | 41 | client := openai.NewClientWithConfig(config) 42 | 43 | openaiResponse, err := client.CreateChatCompletion(ctx, inputToRequest(input, false)) 44 | if err != nil { 45 | return nil, err 46 | } 47 | 48 | response := &model.ChatCompletionResult{ 49 | ID: openaiResponse.ID, 50 | Object: openaiResponse.Object, 51 | Created: int(openaiResponse.Created), 52 | Model: openaiResponse.Model, 53 | Choices: lo.Map(openaiResponse.Choices, func(item openai.ChatCompletionChoice, index int) *model.ChatCompletionChoice { 54 | choice := &model.ChatCompletionChoice{ 55 | Index: item.Index, 56 | Message: mapMessage(item.Message), 57 | FinishReason: lo.ToPtr(model.FinishReason(item.FinishReason)), 58 | } 59 | if item.LogProbs != nil { 60 | choice.LogProbs = new(model.LogProbs) 61 | } 62 | if item.LogProbs != nil && item.LogProbs.Content != nil { 63 | choice.LogProbs.Content = logProbsToTokenLogProbs(item.LogProbs.Content) 64 | } 65 | 66 | return choice 67 | }), 68 | SystemFingerprint: lo.ToPtr(openaiResponse.SystemFingerprint), 69 | Usage: &model.Usage{ 70 | PromptTokens: openaiResponse.Usage.PromptTokens, 71 | CompletionTokens: openaiResponse.Usage.CompletionTokens, 72 | TotalTokens: openaiResponse.Usage.TotalTokens, 73 | }, 74 | } 75 | 76 | return response, nil 77 | } 78 | 79 | // Models is the resolver for the models field. 80 | func (r *queryResolver) Models(ctx context.Context, first *int, after *string, last *int, before *string) (*model.ModelConnection, error) { 81 | panic(fmt.Errorf("not implemented: Models - models")) 82 | } 83 | 84 | // CreateChatCompletionStream is the resolver for the createChatCompletionStream field. 85 | func (r *subscriptionResolver) CreateChatCompletionStream(ctx context.Context, input model.CreateChatCompletionStreamInput) (<-chan *model.ChatCompletionStreamResult, error) { 86 | config := openai.DefaultConfig(input.APIKey) 87 | if input.BaseURL != nil { 88 | config.BaseURL = *input.BaseURL 89 | } 90 | 91 | client := openai.NewClientWithConfig(config) 92 | 93 | ch := make(chan *model.ChatCompletionStreamResult) 94 | 95 | stream, err := client.CreateChatCompletionStream(ctx, streamInputToRequest(input, true)) 96 | if err != nil { 97 | r.Logger.Error("failed to create chat completion stream", zap.Error(err)) 98 | return nil, err 99 | } 100 | 101 | go func() { 102 | for { 103 | response, err := stream.Recv() 104 | if errors.Is(err, io.EOF) { 105 | r.Logger.Info("stream closed") 106 | 107 | close(ch) 108 | break 109 | } 110 | if err != nil { 111 | r.Logger.Error("failed to receive chat completion stream", zap.Error(err)) 112 | transport.AddSubscriptionError(ctx, &gqlerror.Error{ 113 | Message: err.Error(), 114 | }) 115 | 116 | close(ch) 117 | break 118 | } 119 | 120 | result := &model.ChatCompletionStreamResult{ 121 | ID: response.ID, 122 | Object: response.Object, 123 | Created: int(response.Created), 124 | Model: response.Model, 125 | Choices: lo.Map(response.Choices, func(item openai.ChatCompletionStreamChoice, index int) *model.ChatCompletionStreamChunkChoice { 126 | choice := &model.ChatCompletionStreamChunkChoice{ 127 | Index: item.Index, 128 | Delta: mapDelta(item.Delta), 129 | FinishReason: lo.ToPtr(model.FinishReason(item.FinishReason)), 130 | } 131 | 132 | return choice 133 | }), 134 | SystemFingerprint: lo.ToPtr(response.SystemFingerprint), 135 | } 136 | if response.Usage != nil { 137 | result.Usage = &model.Usage{ 138 | PromptTokens: response.Usage.PromptTokens, 139 | CompletionTokens: response.Usage.CompletionTokens, 140 | TotalTokens: response.Usage.TotalTokens, 141 | } 142 | } 143 | 144 | ch <- result 145 | } 146 | }() 147 | 148 | return ch, nil 149 | } 150 | 151 | // Mutation returns generated.MutationResolver implementation. 152 | func (r *Resolver) Mutation() generated.MutationResolver { return &mutationResolver{r} } 153 | 154 | // Query returns generated.QueryResolver implementation. 155 | func (r *Resolver) Query() generated.QueryResolver { return &queryResolver{r} } 156 | 157 | // Subscription returns generated.SubscriptionResolver implementation. 158 | func (r *Resolver) Subscription() generated.SubscriptionResolver { return &subscriptionResolver{r} } 159 | 160 | type mutationResolver struct{ *Resolver } 161 | type queryResolver struct{ *Resolver } 162 | type subscriptionResolver struct{ *Resolver } 163 | -------------------------------------------------------------------------------- /internal/graph/openai/resolvers/common.go: -------------------------------------------------------------------------------- 1 | package resolvers 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "github.com/lingticio/llmg/internal/graph/openai/model" 7 | "github.com/nekomeowww/fo" 8 | "github.com/samber/lo" 9 | "github.com/sashabaranov/go-openai" 10 | ) 11 | 12 | func inputToRequest(input model.CreateChatCompletionInput, stream bool) openai.ChatCompletionRequest { 13 | request := openai.ChatCompletionRequest{ 14 | Model: input.Model, 15 | Stream: stream, 16 | Messages: lo.Map(input.Messages, func(item *model.ChatCompletionMessageInput, index int) openai.ChatCompletionMessage { 17 | message := openai.ChatCompletionMessage{ 18 | Role: item.Role, 19 | } 20 | 21 | if item.Content != nil { 22 | message.Content = *item.Content 23 | } else if item.MultiContent != nil { 24 | message.MultiContent = make([]openai.ChatMessagePart, 0) 25 | for _, part := range item.MultiContent { 26 | openaiPart := openai.ChatMessagePart{ 27 | Type: openai.ChatMessagePartType(part.Type), 28 | } 29 | 30 | switch part.Type { 31 | case string(openai.ChatMessagePartTypeText): 32 | if part.Text != nil { 33 | openaiPart.Text = *part.Text 34 | } 35 | case string(openai.ChatMessagePartTypeImageURL): 36 | openaiPart.ImageURL = &openai.ChatMessageImageURL{ 37 | URL: part.ImageURL.URL, 38 | } 39 | if part.ImageURL.Detail != nil { 40 | openaiPart.ImageURL.Detail = openai.ImageURLDetail(*part.ImageURL.Detail) 41 | } else { 42 | openaiPart.ImageURL.Detail = openai.ImageURLDetailAuto 43 | } 44 | } 45 | 46 | message.MultiContent = append(message.MultiContent, openaiPart) 47 | } 48 | } 49 | 50 | if item.Name != nil { 51 | message.Name = *item.Name 52 | } 53 | if item.ToolCalls != nil { 54 | message.ToolCalls = make([]openai.ToolCall, 0) 55 | 56 | for _, toolCall := range item.ToolCalls { 57 | openaiToolCall := openai.ToolCall{ 58 | ID: toolCall.ID, 59 | Type: openai.ToolType(toolCall.Type), 60 | Function: openai.FunctionCall{ 61 | Name: toolCall.Function.Name, 62 | Arguments: toolCall.Function.Arguments, 63 | }, 64 | } 65 | 66 | message.ToolCalls = append(message.ToolCalls, openaiToolCall) 67 | } 68 | } 69 | if item.ToolCallID != nil { 70 | message.ToolCallID = *item.ToolCallID 71 | } 72 | 73 | return message 74 | }), 75 | } 76 | if input.MaxTokens != nil { 77 | request.MaxTokens = *input.MaxTokens 78 | } 79 | if input.Temperature != nil { 80 | request.Temperature = float32(*input.Temperature) 81 | } 82 | if input.TopP != nil { 83 | request.TopP = float32(*input.TopP) 84 | } 85 | if input.N != nil { 86 | request.N = *input.N 87 | } 88 | if input.Stop != nil { 89 | request.Stop = lo.Map(lo.Filter(input.Stop, func(item *string, index int) bool { 90 | return item != nil 91 | }), func(item *string, index int) string { 92 | return *item 93 | }) 94 | } 95 | if input.PresencePenalty != nil { 96 | request.PresencePenalty = float32(*input.PresencePenalty) 97 | } 98 | if input.ResponseFormat != nil { 99 | request.ResponseFormat = &openai.ChatCompletionResponseFormat{ 100 | Type: openai.ChatCompletionResponseFormatType(input.ResponseFormat.Type), 101 | } 102 | if input.ResponseFormat.JSONSchema != nil { 103 | request.ResponseFormat.JSONSchema = &openai.ChatCompletionResponseFormatJSONSchema{ 104 | Name: input.ResponseFormat.JSONSchema.Name, 105 | Description: input.ResponseFormat.JSONSchema.Description, 106 | } 107 | if input.ResponseFormat.JSONSchema.Schema != nil { 108 | request.ResponseFormat.JSONSchema.Schema = json.RawMessage(fo.May(json.Marshal(input.ResponseFormat.JSONSchema.Schema))) 109 | } 110 | if input.ResponseFormat.JSONSchema.Strict != nil { 111 | request.ResponseFormat.JSONSchema.Strict = *input.ResponseFormat.JSONSchema.Strict 112 | } 113 | } 114 | } 115 | if input.Seed != nil { 116 | request.Seed = input.Seed 117 | } 118 | if input.FrequencyPenalty != nil { 119 | request.FrequencyPenalty = float32(*input.FrequencyPenalty) 120 | } 121 | if input.LogitBias != nil { 122 | request.LogitBias = lo.FromEntries( 123 | lo.Map( 124 | lo.Entries(input.LogitBias), 125 | func(item lo.Entry[string, any], index int) lo.Entry[string, int] { 126 | value, ok := item.Value.(int) 127 | if !ok { 128 | return lo.Entry[string, int]{ 129 | Key: item.Key, 130 | } 131 | } 132 | 133 | return lo.Entry[string, int]{ 134 | Key: item.Key, 135 | Value: value, 136 | } 137 | }, 138 | ), 139 | ) 140 | } 141 | if input.LogProbs != nil { 142 | request.LogProbs = *input.LogProbs 143 | } 144 | if input.TopLogProbs != nil { 145 | request.TopLogProbs = *input.TopLogProbs 146 | } 147 | if input.User != nil { 148 | request.User = *input.User 149 | } 150 | if input.Tools != nil { 151 | request.Tools = make([]openai.Tool, 0) 152 | 153 | for _, tool := range input.Tools { 154 | openaiTool := openai.Tool{ 155 | Type: openai.ToolType(tool.Type), 156 | Function: &openai.FunctionDefinition{ 157 | Name: tool.Function.Name, 158 | Description: tool.Function.Description, 159 | Parameters: tool.Function.Parameters, 160 | }, 161 | } 162 | 163 | request.Tools = append(request.Tools, openaiTool) 164 | } 165 | } 166 | if input.ToolChoice != nil { 167 | request.ToolChoice = *input.ToolChoice 168 | } 169 | if input.ParallelToolCalls != nil { 170 | request.ParallelToolCalls = *input.ParallelToolCalls 171 | } 172 | 173 | return request 174 | } 175 | 176 | func streamInputToRequest(input model.CreateChatCompletionStreamInput, stream bool) openai.ChatCompletionRequest { 177 | request := openai.ChatCompletionRequest{ 178 | Model: input.Model, 179 | Stream: stream, 180 | Messages: lo.Map(input.Messages, func(item *model.ChatCompletionMessageInput, index int) openai.ChatCompletionMessage { 181 | message := openai.ChatCompletionMessage{ 182 | Role: item.Role, 183 | } 184 | 185 | if item.Content != nil { 186 | message.Content = *item.Content 187 | } else if item.MultiContent != nil { 188 | message.MultiContent = make([]openai.ChatMessagePart, 0) 189 | for _, part := range item.MultiContent { 190 | openaiPart := openai.ChatMessagePart{ 191 | Type: openai.ChatMessagePartType(part.Type), 192 | } 193 | 194 | switch part.Type { 195 | case string(openai.ChatMessagePartTypeText): 196 | if part.Text != nil { 197 | openaiPart.Text = *part.Text 198 | } 199 | case string(openai.ChatMessagePartTypeImageURL): 200 | openaiPart.ImageURL = &openai.ChatMessageImageURL{ 201 | URL: part.ImageURL.URL, 202 | } 203 | if part.ImageURL.Detail != nil { 204 | openaiPart.ImageURL.Detail = openai.ImageURLDetail(*part.ImageURL.Detail) 205 | } else { 206 | openaiPart.ImageURL.Detail = openai.ImageURLDetailAuto 207 | } 208 | } 209 | 210 | message.MultiContent = append(message.MultiContent, openaiPart) 211 | } 212 | } 213 | 214 | if item.Name != nil { 215 | message.Name = *item.Name 216 | } 217 | if item.ToolCalls != nil { 218 | message.ToolCalls = make([]openai.ToolCall, 0) 219 | 220 | for _, toolCall := range item.ToolCalls { 221 | openaiToolCall := openai.ToolCall{ 222 | ID: toolCall.ID, 223 | Type: openai.ToolType(toolCall.Type), 224 | Function: openai.FunctionCall{ 225 | Name: toolCall.Function.Name, 226 | Arguments: toolCall.Function.Arguments, 227 | }, 228 | } 229 | 230 | message.ToolCalls = append(message.ToolCalls, openaiToolCall) 231 | } 232 | } 233 | if item.ToolCallID != nil { 234 | message.ToolCallID = *item.ToolCallID 235 | } 236 | 237 | return message 238 | }), 239 | } 240 | if input.MaxTokens != nil { 241 | request.MaxTokens = *input.MaxTokens 242 | } 243 | if input.Temperature != nil { 244 | request.Temperature = float32(*input.Temperature) 245 | } 246 | if input.TopP != nil { 247 | request.TopP = float32(*input.TopP) 248 | } 249 | if input.N != nil { 250 | request.N = *input.N 251 | } 252 | if input.Stop != nil { 253 | request.Stop = lo.Map(lo.Filter(input.Stop, func(item *string, index int) bool { 254 | return item != nil 255 | }), func(item *string, index int) string { 256 | return *item 257 | }) 258 | } 259 | if input.PresencePenalty != nil { 260 | request.PresencePenalty = float32(*input.PresencePenalty) 261 | } 262 | if input.ResponseFormat != nil { 263 | request.ResponseFormat = &openai.ChatCompletionResponseFormat{ 264 | Type: openai.ChatCompletionResponseFormatType(input.ResponseFormat.Type), 265 | } 266 | if input.ResponseFormat.JSONSchema != nil { 267 | request.ResponseFormat.JSONSchema = &openai.ChatCompletionResponseFormatJSONSchema{ 268 | Name: input.ResponseFormat.JSONSchema.Name, 269 | Description: input.ResponseFormat.JSONSchema.Description, 270 | } 271 | if input.ResponseFormat.JSONSchema.Schema != nil { 272 | request.ResponseFormat.JSONSchema.Schema = json.RawMessage(fo.May(json.Marshal(input.ResponseFormat.JSONSchema.Schema))) 273 | } 274 | if input.ResponseFormat.JSONSchema.Strict != nil { 275 | request.ResponseFormat.JSONSchema.Strict = *input.ResponseFormat.JSONSchema.Strict 276 | } 277 | } 278 | } 279 | if input.Seed != nil { 280 | request.Seed = input.Seed 281 | } 282 | if input.FrequencyPenalty != nil { 283 | request.FrequencyPenalty = float32(*input.FrequencyPenalty) 284 | } 285 | if input.LogitBias != nil { 286 | request.LogitBias = lo.FromEntries( 287 | lo.Map( 288 | lo.Entries(input.LogitBias), 289 | func(item lo.Entry[string, any], index int) lo.Entry[string, int] { 290 | value, ok := item.Value.(int) 291 | if !ok { 292 | return lo.Entry[string, int]{ 293 | Key: item.Key, 294 | } 295 | } 296 | 297 | return lo.Entry[string, int]{ 298 | Key: item.Key, 299 | Value: value, 300 | } 301 | }, 302 | ), 303 | ) 304 | } 305 | if input.LogProbs != nil { 306 | request.LogProbs = *input.LogProbs 307 | } 308 | if input.TopLogProbs != nil { 309 | request.TopLogProbs = *input.TopLogProbs 310 | } 311 | if input.User != nil { 312 | request.User = *input.User 313 | } 314 | if input.Tools != nil { 315 | request.Tools = make([]openai.Tool, 0) 316 | 317 | for _, tool := range input.Tools { 318 | openaiTool := openai.Tool{ 319 | Type: openai.ToolType(tool.Type), 320 | Function: &openai.FunctionDefinition{ 321 | Name: tool.Function.Name, 322 | Description: tool.Function.Description, 323 | Parameters: tool.Function.Parameters, 324 | }, 325 | } 326 | 327 | request.Tools = append(request.Tools, openaiTool) 328 | } 329 | } 330 | if input.ToolChoice != nil { 331 | request.ToolChoice = *input.ToolChoice 332 | } 333 | if input.StreamOptions != nil { 334 | request.StreamOptions = &openai.StreamOptions{} 335 | if input.StreamOptions.IncludeUsage != nil { 336 | request.StreamOptions.IncludeUsage = *input.StreamOptions.IncludeUsage 337 | } 338 | } 339 | if input.ParallelToolCalls != nil { 340 | request.ParallelToolCalls = *input.ParallelToolCalls 341 | } 342 | 343 | return request 344 | } 345 | 346 | func multiContentToParts(multiContent []openai.ChatMessagePart) []model.ChatCompletionMessageContentPart { 347 | return lo.Map(multiContent, func(item openai.ChatMessagePart, index int) model.ChatCompletionMessageContentPart { 348 | switch item.Type { 349 | case openai.ChatMessagePartTypeText: 350 | return model.ChatCompletionContentPartText{ 351 | Text: item.Text, 352 | } 353 | case openai.ChatMessagePartTypeImageURL: 354 | if item.ImageURL == nil { 355 | return model.ChatCompletionContentPartText{} 356 | } 357 | 358 | return model.ChatCompletionContentPartImage{ 359 | ImageURL: &model.ChatCompletionContentPartImageURL{ 360 | URL: item.ImageURL.URL, 361 | Detail: lo.ToPtr(model.ImageDetail(item.ImageURL.Detail)), 362 | }, 363 | } 364 | default: 365 | return model.ChatCompletionContentPartText{} 366 | } 367 | }) 368 | } 369 | 370 | func logProbsToTokenLogProbs(logProbs []openai.LogProb) []*model.TokenLogProb { 371 | return lo.Map(logProbs, func(item openai.LogProb, index int) *model.TokenLogProb { 372 | return &model.TokenLogProb{ 373 | Token: item.Token, 374 | LogProb: item.LogProb, 375 | Bytes: lo.Map(item.Bytes, func(item byte, index int) int { 376 | return index 377 | }), 378 | TopLogProbs: lo.Map(item.TopLogProbs, func(item openai.TopLogProbs, index int) *model.TopLogProb { 379 | return &model.TopLogProb{ 380 | Token: item.Token, 381 | LogProb: item.LogProb, 382 | Bytes: lo.Map(item.Bytes, func(item byte, index int) int { 383 | return index 384 | }), 385 | } 386 | }), 387 | } 388 | }) 389 | } 390 | 391 | func mapMessage(message openai.ChatCompletionMessage) model.ChatCompletionMessage { 392 | switch message.Role { 393 | case openai.ChatMessageRoleSystem: 394 | systemMessage := model.ChatCompletionSystemMessage{ 395 | Role: message.Role, 396 | Content: model.ChatCompletionTextContent{ 397 | Text: message.Content, 398 | }, 399 | Name: lo.ToPtr(message.Name), 400 | } 401 | 402 | return systemMessage 403 | case openai.ChatMessageRoleAssistant: 404 | assistantMessage := model.ChatCompletionAssistantMessage{ 405 | Role: message.Role, 406 | Name: lo.ToPtr(message.Name), 407 | ToolCallID: lo.ToPtr(message.ToolCallID), 408 | } 409 | if message.Content != "" { 410 | assistantMessage.Content = model.ChatCompletionTextContent{ 411 | Text: message.Content, 412 | } 413 | } 414 | if len(message.MultiContent) > 0 { 415 | assistantMessage.Content = model.ChatCompletionArrayContent{ 416 | Parts: multiContentToParts(message.MultiContent), 417 | } 418 | } 419 | if message.ToolCalls != nil { 420 | assistantMessage.ToolCalls = lo.Map(message.ToolCalls, func(item openai.ToolCall, index int) *model.ChatCompletionMessageToolCall { 421 | toolCall := model.ChatCompletionMessageToolCall{ 422 | ID: item.ID, 423 | Function: &model.FunctionCall{ 424 | Name: item.Function.Name, 425 | Arguments: item.Function.Arguments, 426 | }, 427 | Type: string(item.Type), 428 | } 429 | 430 | return &toolCall 431 | }) 432 | } 433 | 434 | return assistantMessage 435 | case openai.ChatMessageRoleUser: 436 | userMessage := model.ChatCompletionUserMessage{ 437 | Role: message.Role, 438 | Name: lo.ToPtr(message.Name), 439 | } 440 | if message.Content != "" { 441 | userMessage.Content = model.ChatCompletionTextContent{ 442 | Text: message.Content, 443 | } 444 | } 445 | if len(message.MultiContent) > 0 { 446 | userMessage.Content = model.ChatCompletionArrayContent{ 447 | Parts: multiContentToParts(message.MultiContent), 448 | } 449 | } 450 | 451 | return userMessage 452 | default: 453 | return model.ChatCompletionUserMessage{} 454 | } 455 | } 456 | 457 | func mapDelta(message openai.ChatCompletionStreamChoiceDelta) *model.ChatCompletionStreamResponseDelta { 458 | delta := &model.ChatCompletionStreamResponseDelta{ 459 | Role: message.Role, 460 | Content: message.Content, 461 | } 462 | if message.FunctionCall != nil { 463 | delta.FunctionCall = &model.FunctionCall{ 464 | Name: message.FunctionCall.Name, 465 | Arguments: message.FunctionCall.Arguments, 466 | } 467 | } 468 | if message.ToolCalls != nil { 469 | delta.ToolCalls = lo.Map(message.ToolCalls, func(item openai.ToolCall, index int) *model.ChatCompletionMessageToolCallChunk { 470 | toolCall := model.ChatCompletionMessageToolCallChunk{ 471 | Index: item.Index, 472 | ID: item.ID, 473 | Type: string(item.Type), 474 | Function: &model.FunctionCallChunk{ 475 | Name: item.Function.Name, 476 | Arguments: item.Function.Arguments, 477 | }, 478 | } 479 | 480 | return &toolCall 481 | }) 482 | } 483 | 484 | return delta 485 | } 486 | -------------------------------------------------------------------------------- /internal/graph/openai/resolvers/resolver.go: -------------------------------------------------------------------------------- 1 | package resolvers 2 | 3 | import "github.com/nekomeowww/xo/logger" 4 | 5 | // This file will not be regenerated automatically. 6 | // 7 | // It serves as dependency injection for your app, add any dependencies you require here. 8 | 9 | type Resolver struct { 10 | Logger *logger.Logger 11 | } 12 | -------------------------------------------------------------------------------- /internal/graph/server/middlewares/headers.go: -------------------------------------------------------------------------------- 1 | package middlewares 2 | 3 | import ( 4 | "context" 5 | "strings" 6 | 7 | "github.com/labstack/echo/v4" 8 | ) 9 | 10 | type ContextKey string 11 | 12 | const ( 13 | ContextKeyHeaderAuthorizationAPIKey ContextKey = "header-authorization-api-key" 14 | ContextKeyHeaderXBaseURL ContextKey = "header-x-base-url" 15 | ) 16 | 17 | func HeaderAPIKey(next echo.HandlerFunc) echo.HandlerFunc { 18 | return func(c echo.Context) error { 19 | apiKey := c.Request().Header.Get("X-Api-Key") 20 | if apiKey == "" { 21 | auth := c.Request().Header.Get("Authorization") 22 | apiKey = strings.TrimPrefix(auth, "Bearer ") 23 | } 24 | 25 | c.SetRequest(c.Request().WithContext(context.WithValue(c.Request().Context(), ContextKeyHeaderAuthorizationAPIKey, apiKey))) 26 | 27 | return next(c) 28 | } 29 | } 30 | 31 | func APIKeyFromContext(ctx context.Context) string { 32 | apiKey, _ := ctx.Value(ContextKeyHeaderAuthorizationAPIKey).(string) 33 | return apiKey 34 | } 35 | 36 | func HeaderXBaseURL(next echo.HandlerFunc) echo.HandlerFunc { 37 | return func(c echo.Context) error { 38 | baseURL := c.Request().Header.Get("X-Base-Url") 39 | 40 | c.SetRequest(c.Request().WithContext(context.WithValue(c.Request().Context(), ContextKeyHeaderXBaseURL, baseURL))) 41 | 42 | return next(c) 43 | } 44 | } 45 | 46 | func XBaseURLFromContext(ctx context.Context) string { 47 | baseURL, _ := ctx.Value(ContextKeyHeaderXBaseURL).(string) 48 | return baseURL 49 | } 50 | -------------------------------------------------------------------------------- /internal/graph/server/server.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "net" 8 | "net/http" 9 | "time" 10 | 11 | "github.com/labstack/echo/v4" 12 | "github.com/labstack/echo/v4/middleware" 13 | "github.com/lingticio/llmg/internal/configs" 14 | "github.com/lingticio/llmg/internal/graph/openai" 15 | "github.com/lingticio/llmg/internal/graph/server/middlewares" 16 | "github.com/nekomeowww/xo/logger" 17 | "go.uber.org/fx" 18 | "go.uber.org/zap" 19 | ) 20 | 21 | func Modules() fx.Option { 22 | return fx.Options( 23 | fx.Provide(openai.NewGraphQLHandler()), 24 | fx.Provide(NewServer()), 25 | ) 26 | } 27 | 28 | type NewServerParams struct { 29 | fx.In 30 | 31 | Lifecycle fx.Lifecycle 32 | Logger *logger.Logger 33 | Config *configs.Config 34 | 35 | OpenAIHandler *openai.GraphQLHandler 36 | } 37 | 38 | type Server struct { 39 | *http.Server 40 | } 41 | 42 | func NewServer() func(params NewServerParams) *Server { 43 | return func(params NewServerParams) *Server { 44 | e := echo.New() 45 | 46 | e.Use(middlewares.HeaderXBaseURL) 47 | e.Use(middlewares.HeaderAPIKey) 48 | e.Use(middleware.CORSWithConfig(middleware.CORSConfig{ 49 | AllowOriginFunc: func(origin string) (bool, error) { 50 | return true, nil 51 | }, 52 | AllowHeaders: []string{"Origin", "Content-Length", "Content-Type"}, 53 | AllowMethods: []string{"GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD"}, 54 | MaxAge: 60 * 60 * 24 * 7, //nolint:mnd 55 | })) 56 | 57 | params.OpenAIHandler.InstallForEcho("/v1/openai/query", e) 58 | params.OpenAIHandler.InstallPlaygroundForEcho("/v1/openai/query", "/v1/openai/", e) 59 | 60 | for _, v := range e.Routes() { 61 | params.Logger.Debug("registered route", zap.String("method", v.Method), zap.String("path", v.Path)) 62 | } 63 | 64 | server := &http.Server{ 65 | Addr: params.Config.GraphQL.Addr, 66 | Handler: e, 67 | ReadHeaderTimeout: time.Minute, 68 | } 69 | 70 | params.Lifecycle.Append(fx.Hook{ 71 | OnStop: func(ctx context.Context) error { 72 | closeCtx, cancel := context.WithTimeout(context.Background(), time.Minute) 73 | defer cancel() 74 | 75 | if err := server.Shutdown(closeCtx); err != nil && !errors.Is(err, http.ErrServerClosed) { 76 | params.Logger.Error("shutdown graphql server failed", zap.Error(err)) 77 | return err 78 | } 79 | 80 | return nil 81 | }, 82 | }) 83 | 84 | return &Server{ 85 | Server: server, 86 | } 87 | } 88 | } 89 | 90 | func Run() func(logger *logger.Logger, server *Server) error { 91 | return func(logger *logger.Logger, server *Server) error { 92 | logger.Info("starting graphql server...") 93 | 94 | listener, err := net.Listen("tcp", server.Addr) 95 | if err != nil { 96 | return fmt.Errorf("failed to listen %s: %w", server.Addr, err) 97 | } 98 | 99 | go func() { 100 | err = server.Serve(listener) 101 | if err != nil && !errors.Is(err, http.ErrServerClosed) { 102 | logger.Fatal(err.Error()) 103 | } 104 | }() 105 | 106 | logger.Info("graphql server listening...", zap.String("addr", server.Addr)) 107 | 108 | return nil 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /internal/grpc/servers/apiserver/grpc_gateway.go: -------------------------------------------------------------------------------- 1 | package apiserver 2 | 3 | import ( 4 | "context" 5 | "encoding/gob" 6 | "fmt" 7 | "net" 8 | "net/http" 9 | "time" 10 | 11 | "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" 12 | "github.com/labstack/echo/v4" 13 | "github.com/labstack/echo/v4/middleware" 14 | "github.com/nekomeowww/xo/logger" 15 | "go.uber.org/fx" 16 | "go.uber.org/zap" 17 | "google.golang.org/grpc" 18 | "google.golang.org/grpc/credentials/insecure" 19 | 20 | "github.com/lingticio/llmg/internal/configs" 21 | "github.com/lingticio/llmg/internal/grpc/servers/interceptors" 22 | "github.com/lingticio/llmg/internal/grpc/servers/middlewares" 23 | grpcpkg "github.com/lingticio/llmg/pkg/util/grpc" 24 | ) 25 | 26 | type NewGatewayServerParams struct { 27 | fx.In 28 | 29 | Lifecycle fx.Lifecycle 30 | Config *configs.Config 31 | Register *grpcpkg.Register 32 | Logger *logger.Logger 33 | } 34 | 35 | type GatewayServer struct { 36 | ListenAddr string 37 | GRPCServerAddr string 38 | 39 | echo *echo.Echo 40 | server *http.Server 41 | } 42 | 43 | func NewGatewayServer() func(params NewGatewayServerParams) (*GatewayServer, error) { 44 | return func(params NewGatewayServerParams) (*GatewayServer, error) { 45 | gob.Register(map[interface{}]interface{}{}) 46 | 47 | e := echo.New() 48 | 49 | e.Use(middlewares.ResponseLog(params.Logger)) 50 | e.Use(middleware.CORSWithConfig(middleware.CORSConfig{ 51 | AllowOrigins: []string{ 52 | "http://localhost:3000", 53 | }, 54 | AllowHeaders: []string{ 55 | echo.HeaderOrigin, 56 | echo.HeaderContentType, 57 | echo.HeaderAccept, 58 | echo.HeaderAuthorization, 59 | }, 60 | })) 61 | e.RouteNotFound("/*", middlewares.NotFound) 62 | 63 | for path, methodHandlers := range params.Register.EchoHandlers { 64 | for method, handler := range methodHandlers { 65 | e.Add(method, path, handler) 66 | } 67 | } 68 | 69 | server := &GatewayServer{ 70 | ListenAddr: params.Config.HTTP.Addr, 71 | GRPCServerAddr: params.Config.Grpc.Addr, 72 | echo: e, 73 | server: &http.Server{ 74 | Addr: params.Config.HTTP.Addr, 75 | Handler: e, 76 | ReadHeaderTimeout: time.Duration(30) * time.Second, 77 | }, 78 | } 79 | 80 | params.Lifecycle.Append(fx.Hook{ 81 | OnStart: func(ctx context.Context) error { 82 | conn, err := grpc.NewClient(params.Config.Grpc.Addr, grpc.WithTransportCredentials(insecure.NewCredentials())) 83 | if err != nil { 84 | return err 85 | } 86 | 87 | gateway, err := grpcpkg.NewGateway(ctx, conn, params.Logger, 88 | grpcpkg.WithServerMuxOptions( 89 | runtime.WithErrorHandler(interceptors.HTTPErrorHandler(params.Logger)), 90 | runtime.WithMetadata(interceptors.MetadataCookie()), 91 | runtime.WithMetadata(interceptors.MetadataAuthorization()), 92 | runtime.WithMetadata(interceptors.MetadataRequestPath()), 93 | ), 94 | grpcpkg.WithHandlers(params.Register.HTTPHandlers...), 95 | ) 96 | if err != nil { 97 | return err 98 | } 99 | 100 | server.echo.Any("/api/v1/*", echo.WrapHandler(gateway)) 101 | 102 | return nil 103 | }, 104 | }) 105 | 106 | return server, nil 107 | } 108 | } 109 | 110 | func RunGatewayServer() func(logger *logger.Logger, server *GatewayServer) error { 111 | return func(logger *logger.Logger, server *GatewayServer) error { 112 | logger.Info("starting http server...") 113 | 114 | listener, err := net.Listen("tcp", server.ListenAddr) 115 | if err != nil { 116 | return fmt.Errorf("failed to listen %s: %v", server.ListenAddr, err) 117 | } 118 | 119 | go func() { 120 | err = server.server.Serve(listener) 121 | if err != nil && err != http.ErrServerClosed { 122 | logger.Fatal(err.Error()) 123 | } 124 | }() 125 | 126 | logger.Info("http server listening...", zap.String("addr", server.ListenAddr)) 127 | 128 | return nil 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /internal/grpc/servers/apiserver/grpc_server.go: -------------------------------------------------------------------------------- 1 | package apiserver 2 | 3 | import ( 4 | "context" 5 | "net" 6 | 7 | "github.com/nekomeowww/xo/logger" 8 | "go.uber.org/fx" 9 | "go.uber.org/zap" 10 | "google.golang.org/grpc" 11 | 12 | "github.com/lingticio/llmg/internal/configs" 13 | "github.com/lingticio/llmg/internal/grpc/servers/interceptors" 14 | grpcpkg "github.com/lingticio/llmg/pkg/util/grpc" 15 | ) 16 | 17 | type NewGRPCServerParams struct { 18 | fx.In 19 | 20 | Lifecycle fx.Lifecycle 21 | Logger *logger.Logger 22 | Config *configs.Config 23 | Register *grpcpkg.Register 24 | } 25 | 26 | type GRPCServer struct { 27 | ListenAddr string 28 | 29 | server *grpc.Server 30 | register *grpcpkg.Register 31 | } 32 | 33 | func NewGRPCServer() func(params NewGRPCServerParams) *GRPCServer { 34 | return func(params NewGRPCServerParams) *GRPCServer { 35 | server := grpc.NewServer( 36 | grpc.ChainUnaryInterceptor( 37 | interceptors.PanicInterceptor(params.Logger), 38 | ), 39 | ) 40 | 41 | params.Lifecycle.Append(fx.Hook{ 42 | OnStop: func(ctx context.Context) error { 43 | params.Logger.Info("gracefully shutting down gRPC server...") 44 | server.GracefulStop() 45 | return nil 46 | }, 47 | }) 48 | 49 | return &GRPCServer{ 50 | ListenAddr: params.Config.Grpc.Addr, 51 | server: server, 52 | register: params.Register, 53 | } 54 | } 55 | } 56 | 57 | func RunGRPCServer() func(logger *logger.Logger, server *GRPCServer) error { 58 | return func(logger *logger.Logger, server *GRPCServer) error { 59 | for _, serviceRegister := range server.register.GrpcServices { 60 | serviceRegister(server.server) 61 | } 62 | 63 | l, err := net.Listen("tcp", server.ListenAddr) 64 | if err != nil { 65 | return err 66 | } 67 | 68 | go func() { 69 | err := server.server.Serve(l) 70 | if err != nil && err != grpc.ErrServerStopped { 71 | logger.Fatal("failed to serve gRPC server", zap.Error(err)) 72 | } 73 | }() 74 | 75 | logger.Info("gRPC server started", zap.String("listen_addr", server.ListenAddr)) 76 | 77 | return nil 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /internal/grpc/servers/interceptors/apikey.go: -------------------------------------------------------------------------------- 1 | package interceptors 2 | 3 | import "context" 4 | 5 | func OpenAIStyleAPIKeyFromContext(ctx context.Context) (string, error) { 6 | authorization, err := AuthorizationFromContext(ctx) 7 | if err != nil { 8 | return "", err 9 | } 10 | 11 | return BearerFromAuthorization(authorization) 12 | } 13 | -------------------------------------------------------------------------------- /internal/grpc/servers/interceptors/authorization.go: -------------------------------------------------------------------------------- 1 | package interceptors 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "net/http" 7 | 8 | "github.com/lingticio/llmg/pkg/apierrors" 9 | "google.golang.org/grpc/metadata" 10 | ) 11 | 12 | func MetadataAuthorization() func(context.Context, *http.Request) metadata.MD { 13 | return func(ctx context.Context, r *http.Request) metadata.MD { 14 | md := metadata.MD{} 15 | 16 | authorization := r.Header.Get("Authorization") 17 | if authorization != "" { 18 | md.Append("header-authorization", authorization) 19 | } 20 | 21 | return md 22 | } 23 | } 24 | 25 | func AuthorizationFromMetadata(md metadata.MD) (string, error) { 26 | values := md.Get("header-authorization") 27 | if len(values) == 0 { 28 | return "", nil 29 | } 30 | 31 | return values[0], nil 32 | } 33 | 34 | func AuthorizationFromContext(ctx context.Context) (string, error) { 35 | md, ok := metadata.FromIncomingContext(ctx) 36 | if !ok { 37 | return "", apierrors.NewErrInternal().WithError(errors.New("failed to get metadata from context")).WithCaller().AsStatus() 38 | } 39 | 40 | return AuthorizationFromMetadata(md) 41 | } 42 | 43 | func BearerFromAuthorization(authorization string) (string, error) { 44 | if authorization == "" { 45 | return "", nil 46 | } 47 | 48 | if len(authorization) < 7 || authorization[:7] != "Bearer " { 49 | return "", apierrors.NewBadRequest().WithDetail("malformed Authorization header, expected Bearer token").AsStatus() 50 | } 51 | 52 | return authorization[7:], nil 53 | } 54 | -------------------------------------------------------------------------------- /internal/grpc/servers/interceptors/cookie.go: -------------------------------------------------------------------------------- 1 | package interceptors 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "errors" 7 | "net/http" 8 | "strings" 9 | 10 | "github.com/lingticio/llmg/pkg/apierrors" 11 | "github.com/nekomeowww/fo" 12 | "google.golang.org/grpc/metadata" 13 | ) 14 | 15 | func MetadataCookie() func(context.Context, *http.Request) metadata.MD { 16 | return func(ctx context.Context, r *http.Request) metadata.MD { 17 | md := metadata.MD{} 18 | 19 | for _, cookie := range r.Cookies() { 20 | md.Append("header-cookie-"+cookie.Name, string(fo.May(json.Marshal(http.Cookie{ 21 | Name: cookie.Name, 22 | Value: cookie.Value, 23 | Path: cookie.Path, 24 | Domain: cookie.Domain, 25 | Expires: cookie.Expires, 26 | RawExpires: cookie.RawExpires, 27 | MaxAge: cookie.MaxAge, 28 | Secure: cookie.Secure, 29 | HttpOnly: cookie.HttpOnly, 30 | SameSite: cookie.SameSite, 31 | Raw: cookie.Raw, 32 | Unparsed: cookie.Unparsed, 33 | })))) 34 | } 35 | 36 | return md 37 | } 38 | } 39 | 40 | type Cookies []*http.Cookie 41 | 42 | func (c Cookies) Cookie(name string) *http.Cookie { 43 | for _, cookie := range c { 44 | if cookie.Name == name { 45 | return cookie 46 | } 47 | } 48 | 49 | return nil 50 | } 51 | 52 | func CookiesFromContext(ctx context.Context) (Cookies, error) { 53 | md, ok := metadata.FromIncomingContext(ctx) 54 | if !ok { 55 | return nil, apierrors.NewErrInternal().WithError(errors.New("failed to get metadata from context")).WithCaller().AsStatus() 56 | } 57 | 58 | return CookiesFromMetadata(md) 59 | } 60 | 61 | func CookiesFromMetadata(md metadata.MD) (Cookies, error) { 62 | var cookies Cookies 63 | 64 | for k, v := range md { 65 | if len(v) == 0 { 66 | continue 67 | } 68 | if strings.HasPrefix(k, "header-cookie-") { 69 | var cookie http.Cookie 70 | 71 | err := json.Unmarshal([]byte(v[0]), &cookie) 72 | if err != nil { 73 | return nil, err 74 | } 75 | 76 | cookies = append(cookies, &cookie) 77 | } 78 | } 79 | 80 | return cookies, nil 81 | } 82 | -------------------------------------------------------------------------------- /internal/grpc/servers/interceptors/error_handler.go: -------------------------------------------------------------------------------- 1 | package interceptors 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | 9 | "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" 10 | "github.com/nekomeowww/xo/logger" 11 | "go.uber.org/zap" 12 | "google.golang.org/grpc/codes" 13 | "google.golang.org/grpc/status" 14 | 15 | "github.com/lingticio/llmg/apis/jsonapi" 16 | "github.com/lingticio/llmg/pkg/apierrors" 17 | ) 18 | 19 | func handleStatusError(logger *logger.Logger, request *http.Request, s *status.Status, err error) *apierrors.ErrResponse { //nolint:cyclop 20 | switch s.Code() { //nolint 21 | case codes.InvalidArgument: 22 | if len(s.Details()) > 0 { 23 | break 24 | } 25 | 26 | return apierrors.NewErrInvalidArgument().WithDetail(s.Message()).AsResponse() 27 | case codes.Unimplemented: 28 | logger.Error("unimplemented error", zap.Error(err), zap.String("method", request.Method), zap.String("path", request.URL.Path)) 29 | 30 | return apierrors.NewErrNotFound().WithDetail("route not found or method not allowed").AsResponse() 31 | case codes.Internal: 32 | var errorCaller *jsonapi.ErrorCaller 33 | 34 | if len(s.Details()) > 1 { 35 | errorCaller, _ = s.Details()[1].(*jsonapi.ErrorCaller) 36 | } 37 | 38 | fields := []zap.Field{ 39 | zap.Error(err), 40 | zap.String("method", request.Method), 41 | zap.String("path", request.URL.Path), 42 | } 43 | if errorCaller != nil { 44 | fields = append(fields, zap.String("file", fmt.Sprintf("%s:%d", errorCaller.GetFile(), errorCaller.GetLine()))) 45 | fields = append(fields, zap.String("function", errorCaller.GetFunction())) 46 | } 47 | 48 | logger.Error("internal error", fields...) 49 | 50 | return apierrors.NewErrInternal().AsResponse() 51 | case codes.NotFound: 52 | if len(s.Details()) > 0 { 53 | break 54 | } 55 | 56 | logger.Error("unimplemented error", zap.Error(err), zap.String("method", request.Method), zap.String("path", request.URL.Path)) 57 | 58 | return apierrors.NewErrNotFound().WithDetail("route not found or method not allowed").AsResponse() 59 | case codes.Unknown: 60 | logger.Error("unknown error", zap.Error(err), zap.String("method", request.Method), zap.String("path", request.URL.Path)) 61 | 62 | return apierrors.NewErrInternal().AsResponse() 63 | default: 64 | break 65 | } 66 | 67 | errResp := apierrors.NewErrResponse() 68 | if len(s.Details()) > 0 { 69 | detail, ok := s.Details()[0].(*jsonapi.ErrorObject) 70 | if ok { 71 | errResp = errResp.WithError(&apierrors.Error{ 72 | ErrorObject: detail, 73 | }) 74 | } 75 | } 76 | 77 | return errResp 78 | } 79 | 80 | func handleError(logger *logger.Logger, request *http.Request, err error) *apierrors.ErrResponse { 81 | if s, ok := status.FromError(err); ok { 82 | return handleStatusError(logger, request, s, err) 83 | } 84 | 85 | logger.Error("unknown error (probably unhandled)", zap.Error(err)) 86 | 87 | return apierrors.NewErrInternal().AsResponse() 88 | } 89 | 90 | func HTTPErrorHandler(logger *logger.Logger) func(ctx context.Context, _ *runtime.ServeMux, _ runtime.Marshaler, writer http.ResponseWriter, _ *http.Request, err error) { 91 | return func(ctx context.Context, _ *runtime.ServeMux, _ runtime.Marshaler, writer http.ResponseWriter, request *http.Request, err error) { 92 | if err != nil { 93 | errResp := handleError(logger, request, err) 94 | 95 | b, _ := json.Marshal(errResp) 96 | 97 | writer.Header().Set("Content-Type", "application/json") 98 | writer.WriteHeader(errResp.HTTPStatus()) 99 | 100 | _, _ = writer.Write(b) 101 | } 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /internal/grpc/servers/interceptors/panic.go: -------------------------------------------------------------------------------- 1 | package interceptors 2 | 3 | import ( 4 | "context" 5 | "runtime/debug" 6 | 7 | "github.com/lingticio/llmg/pkg/apierrors" 8 | "github.com/nekomeowww/xo/logger" 9 | "go.uber.org/zap" 10 | "google.golang.org/grpc" 11 | ) 12 | 13 | func PanicInterceptor(logger *logger.Logger) grpc.UnaryServerInterceptor { 14 | return func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { //nolint:nonamedreturns 15 | defer func() { 16 | r := recover() 17 | if r != nil { 18 | logger.Error("panicked", zap.Any("err", r), zap.Stack(string(debug.Stack()))) 19 | err = apierrors.NewErrInternal().AsStatus() 20 | resp = nil 21 | } 22 | }() 23 | 24 | return handler(ctx, req) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /internal/grpc/servers/interceptors/request_path.go: -------------------------------------------------------------------------------- 1 | package interceptors 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | 7 | "google.golang.org/grpc/metadata" 8 | ) 9 | 10 | func MetadataRequestPath() func(ctx context.Context, request *http.Request) metadata.MD { 11 | return func(ctx context.Context, request *http.Request) metadata.MD { 12 | return metadata.New(map[string]string{ 13 | "path": request.URL.Path, 14 | }) 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /internal/grpc/servers/llmg/v1/v1.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "net" 7 | "net/http" 8 | 9 | openaiapiv1 "github.com/lingticio/llmg/apis/llmgapi/v1/openai" 10 | "github.com/lingticio/llmg/internal/configs" 11 | "github.com/lingticio/llmg/internal/grpc/services/llmgapi/v1/openai" 12 | "github.com/nekomeowww/xo/logger" 13 | "go.uber.org/fx" 14 | "go.uber.org/zap" 15 | "google.golang.org/grpc" 16 | "google.golang.org/grpc/reflection" 17 | ) 18 | 19 | type NewV1GRPCServerParam struct { 20 | fx.In 21 | 22 | Lifecycle fx.Lifecycle 23 | Config *configs.Config 24 | Logger *logger.Logger 25 | OpenAIService *openai.OpenAIService 26 | } 27 | 28 | type V1GRPCServer struct { 29 | GRPCServer *grpc.Server 30 | Addr string 31 | } 32 | 33 | func NewV1GRPCServer() func(params NewV1GRPCServerParam) *V1GRPCServer { 34 | return func(params NewV1GRPCServerParam) *V1GRPCServer { 35 | grpcServer := grpc.NewServer() 36 | openaiapiv1.RegisterOpenAIServiceServer(grpcServer, params.OpenAIService) 37 | reflection.Register(grpcServer) 38 | 39 | params.Lifecycle.Append(fx.Hook{ 40 | OnStop: func(ctx context.Context) error { 41 | params.Logger.Info("gracefully shutting down v1 gRPC server...") 42 | grpcServer.GracefulStop() 43 | return nil 44 | }, 45 | }) 46 | 47 | return &V1GRPCServer{ 48 | GRPCServer: grpcServer, 49 | Addr: params.Config.Grpc.Addr, 50 | } 51 | } 52 | } 53 | 54 | func Run() func(*logger.Logger, *V1GRPCServer) error { 55 | return func(logger *logger.Logger, server *V1GRPCServer) error { 56 | logger.Info("starting v1 gRPC service...") 57 | 58 | listener, err := net.Listen("tcp", server.Addr) 59 | if err != nil { 60 | return err 61 | } 62 | 63 | go func() { 64 | err := server.GRPCServer.Serve(listener) 65 | if err != nil && !errors.Is(err, http.ErrServerClosed) { 66 | logger.Error("failed to serve v1 gRPC server", zap.Error(err)) 67 | } 68 | }() 69 | 70 | logger.Info("v1 gRPC server listening", zap.String("addr", listener.Addr().String())) 71 | 72 | return nil 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /internal/grpc/servers/middlewares/response_log.go: -------------------------------------------------------------------------------- 1 | package middlewares 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/labstack/echo/v4" 7 | "github.com/nekomeowww/xo/logger" 8 | "go.uber.org/zap" 9 | ) 10 | 11 | func ResponseLog(logger *logger.Logger) echo.MiddlewareFunc { 12 | return func(next echo.HandlerFunc) echo.HandlerFunc { 13 | return func(c echo.Context) error { 14 | start := time.Now() 15 | 16 | err := next(c) 17 | if err != nil { 18 | return err 19 | } 20 | 21 | end := time.Now() 22 | 23 | logger.Debug("", 24 | zap.String("latency", end.Sub(start).String()), 25 | zap.String("path", c.Request().RequestURI), 26 | zap.String("remote", c.Request().RemoteAddr), 27 | zap.String("hosts", c.Request().URL.Host), 28 | zap.Int("status", c.Response().Status), 29 | zap.String("method", c.Request().Method), 30 | ) 31 | 32 | return nil 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /internal/grpc/servers/middlewares/route_not_found.go: -------------------------------------------------------------------------------- 1 | package middlewares 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/labstack/echo/v4" 7 | "github.com/lingticio/llmg/pkg/apierrors" 8 | ) 9 | 10 | func NotFound(c echo.Context) error { 11 | return c.JSON(http.StatusNotFound, apierrors.NewErrNotFound().AsResponse()) 12 | } 13 | -------------------------------------------------------------------------------- /internal/grpc/servers/servers.go: -------------------------------------------------------------------------------- 1 | package servers 2 | 3 | import ( 4 | "github.com/lingticio/llmg/internal/grpc/servers/apiserver" 5 | v1 "github.com/lingticio/llmg/internal/grpc/servers/llmg/v1" 6 | "go.uber.org/fx" 7 | ) 8 | 9 | func Modules() fx.Option { 10 | return fx.Options( 11 | fx.Provide(apiserver.NewGRPCServer()), 12 | fx.Provide(apiserver.NewGatewayServer()), 13 | fx.Provide(v1.NewV1GRPCServer()), 14 | ) 15 | } 16 | -------------------------------------------------------------------------------- /internal/grpc/services/llmgapi/v1/openai/openai.go: -------------------------------------------------------------------------------- 1 | package openai 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "io" 7 | "time" 8 | 9 | "github.com/nekomeowww/xo/logger" 10 | "github.com/samber/lo" 11 | "github.com/sashabaranov/go-openai" 12 | "go.uber.org/fx" 13 | "go.uber.org/zap" 14 | "google.golang.org/grpc/codes" 15 | "google.golang.org/grpc/metadata" 16 | "google.golang.org/grpc/status" 17 | "google.golang.org/protobuf/types/known/timestamppb" 18 | 19 | openaiapiv1 "github.com/lingticio/llmg/apis/llmgapi/v1/openai" 20 | ) 21 | 22 | type NewOpenAIServiceParams struct { 23 | fx.In 24 | 25 | Logger *logger.Logger 26 | } 27 | 28 | type OpenAIService struct { 29 | openaiapiv1.UnimplementedOpenAIServiceServer 30 | 31 | logger *logger.Logger 32 | } 33 | 34 | func NewOpenAIService() func(params NewOpenAIServiceParams) *OpenAIService { 35 | return func(params NewOpenAIServiceParams) *OpenAIService { 36 | return &OpenAIService{ 37 | logger: params.Logger, 38 | } 39 | } 40 | } 41 | 42 | func clientConfigFromContext(ctx context.Context) (openai.ClientConfig, error) { 43 | md, ok := metadata.FromIncomingContext(ctx) 44 | if !ok { 45 | return openai.ClientConfig{}, status.Errorf(codes.Internal, "failed to get metadata") 46 | } 47 | 48 | apiKeys := md.Get("x-api-key") 49 | if len(apiKeys) == 0 { 50 | return openai.ClientConfig{}, status.Errorf(codes.InvalidArgument, "missing API key in x-api-key") 51 | } 52 | 53 | config := openai.DefaultConfig(apiKeys[0]) 54 | 55 | baseURLs := md.Get("x-base-url") 56 | if len(baseURLs) > 0 { 57 | config.BaseURL = baseURLs[0] 58 | } 59 | 60 | return config, nil 61 | } 62 | 63 | func (s *OpenAIService) CreateChatCompletion(ctx context.Context, req *openaiapiv1.CreateChatCompletionRequest) (*openaiapiv1.CreateChatCompletionResponse, error) { 64 | config, err := clientConfigFromContext(ctx) 65 | if err != nil { 66 | return nil, err 67 | } 68 | 69 | client := openai.NewClientWithConfig(config) 70 | 71 | openaiResponse, err := client.CreateChatCompletion(ctx, gRPCRequestToOpenAIRequest(req)) 72 | if err != nil { 73 | return nil, status.Errorf(codes.Internal, "failed to create chat completion: %v", err) 74 | } 75 | 76 | response := &openaiapiv1.CreateChatCompletionResponse{ 77 | Id: openaiResponse.ID, 78 | Object: openaiResponse.Object, 79 | Created: timestamppb.New(time.Unix(openaiResponse.Created, 0)), 80 | Model: openaiResponse.Model, 81 | Choices: lo.Map(openaiResponse.Choices, func(item openai.ChatCompletionChoice, index int) *openaiapiv1.ChatCompletionChoice { 82 | choice := &openaiapiv1.ChatCompletionChoice{ 83 | Index: int64(item.Index), 84 | Message: mapMessage(item.Message), 85 | FinishReason: openaiapiv1.ChatCompletionFinishReason(openaiapiv1.ChatCompletionFinishReason_value[string(item.FinishReason)]), 86 | } 87 | if item.LogProbs != nil { 88 | choice.LogProbs = new(openaiapiv1.ChatCompletionChoiceLogProbs) 89 | } 90 | if item.LogProbs != nil && item.LogProbs.Content != nil { 91 | choice.LogProbs.Content = logProbsToTokenLogProbs(item.LogProbs.Content) 92 | } 93 | 94 | return choice 95 | }), 96 | ServiceTier: lo.ToPtr(req.GetServiceTier()), 97 | SystemFingerprint: lo.ToPtr(openaiResponse.SystemFingerprint), 98 | Usage: &openaiapiv1.ChatCompletionUsage{ 99 | PromptTokens: int64(openaiResponse.Usage.PromptTokens), 100 | CompletionTokens: int64(openaiResponse.Usage.CompletionTokens), 101 | TotalTokens: int64(openaiResponse.Usage.TotalTokens), 102 | }, 103 | } 104 | 105 | return response, nil 106 | } 107 | 108 | func (s *OpenAIService) CreateChatCompletionStream(req *openaiapiv1.CreateChatCompletionStreamRequest, server openaiapiv1.OpenAIService_CreateChatCompletionStreamServer) error { 109 | config, err := clientConfigFromContext(server.Context()) 110 | if err != nil { 111 | return err 112 | } 113 | 114 | client := openai.NewClientWithConfig(config) 115 | 116 | stream, err := client.CreateChatCompletionStream(server.Context(), gRPCStreamRequestToOpenAIRequest(req)) 117 | if err != nil { 118 | return status.Errorf(codes.Internal, "failed to create chat completion stream: %v", err) 119 | } 120 | 121 | for { 122 | response, err := stream.Recv() 123 | if errors.Is(err, io.EOF) { 124 | s.logger.Info("stream closed") 125 | break 126 | } 127 | if err != nil { 128 | s.logger.Error("failed to receive chat completion stream", zap.Error(err)) 129 | return status.Errorf(codes.Internal, "failed to receive chat completion stream: %v", err) 130 | } 131 | 132 | chunkResponse := &openaiapiv1.CreateChatCompletionStreamResponse{ 133 | Id: response.ID, 134 | Object: response.Object, 135 | Created: timestamppb.New(time.Unix(response.Created, 0)), 136 | Model: response.Model, 137 | Choices: lo.Map(response.Choices, func(item openai.ChatCompletionStreamChoice, index int) *openaiapiv1.ChatCompletionChunkChoice { 138 | choice := &openaiapiv1.ChatCompletionChunkChoice{ 139 | Index: int64(item.Index), 140 | Delta: mapDelta(item.Delta), 141 | FinishReason: lo.ToPtr(mapOpenAIFinishedReasonToChatCompletionFinishReason[item.FinishReason]), 142 | } 143 | 144 | return choice 145 | }), 146 | SystemFingerprint: lo.ToPtr(response.SystemFingerprint), 147 | } 148 | if response.Usage != nil { 149 | chunkResponse.Usage = &openaiapiv1.ChatCompletionUsage{ 150 | PromptTokens: int64(response.Usage.PromptTokens), 151 | CompletionTokens: int64(response.Usage.CompletionTokens), 152 | TotalTokens: int64(response.Usage.TotalTokens), 153 | } 154 | } 155 | 156 | if err := server.Send(chunkResponse); err != nil { 157 | s.logger.Error("failed to send chat completion stream", zap.Error(err)) 158 | } 159 | } 160 | 161 | return nil 162 | } 163 | -------------------------------------------------------------------------------- /internal/grpc/services/services.go: -------------------------------------------------------------------------------- 1 | package services 2 | 3 | import ( 4 | "github.com/lingticio/llmg/internal/grpc/services/llmgapi/v1/openai" 5 | "go.uber.org/fx" 6 | ) 7 | 8 | func Modules() fx.Option { 9 | return fx.Options( 10 | fx.Provide(openai.NewOpenAIService()), 11 | ) 12 | } 13 | -------------------------------------------------------------------------------- /internal/libs/libs.go: -------------------------------------------------------------------------------- 1 | package libs 2 | 3 | import ( 4 | "go.uber.org/fx" 5 | ) 6 | 7 | func Modules() fx.Option { 8 | return fx.Options( 9 | fx.Provide(NewLogger()), 10 | ) 11 | } 12 | -------------------------------------------------------------------------------- /internal/libs/logger.go: -------------------------------------------------------------------------------- 1 | package libs 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "time" 7 | 8 | "github.com/nekomeowww/xo" 9 | "github.com/nekomeowww/xo/logger" 10 | "github.com/nekomeowww/xo/logger/loki" 11 | "github.com/samber/lo" 12 | "go.uber.org/zap" 13 | "go.uber.org/zap/zapcore" 14 | 15 | "github.com/lingticio/llmg/internal/configs" 16 | "github.com/lingticio/llmg/internal/meta" 17 | ) 18 | 19 | func NewLogger() func(config *configs.Config) (*logger.Logger, error) { 20 | return func(config *configs.Config) (*logger.Logger, error) { 21 | logLevel, err := logger.ReadLogLevelFromEnv() 22 | if err != nil { 23 | logLevel = zapcore.InfoLevel 24 | } 25 | 26 | var isFatalLevel bool 27 | if logLevel == zapcore.FatalLevel { 28 | isFatalLevel = true 29 | logLevel = zapcore.InfoLevel 30 | } 31 | 32 | logFormat, readFormatError := logger.ReadLogFormatFromEnv() 33 | 34 | logger, err := logger.NewLogger( 35 | logger.WithLevel(logLevel), 36 | logger.WithAppName(config.Meta.App), 37 | logger.WithNamespace(config.Meta.Namespace), 38 | logger.WithLogFilePath(xo.RelativePathBasedOnPwdOf("./logs/"+config.Meta.App)), 39 | logger.WithFormat(logFormat), 40 | logger.WithLokiRemoteConfig(lo.Ternary(os.Getenv("LOG_LOKI_REMOTE_URL") != "", &loki.Config{ 41 | Url: os.Getenv("LOG_LOKI_REMOTE_URL"), 42 | BatchMaxSize: 2000, //nolint:mnd 43 | BatchMaxWait: 10 * time.Second, //nolint:mnd 44 | }, nil)), 45 | ) 46 | if err != nil { 47 | return nil, fmt.Errorf("failed to create logger: %w", err) 48 | } 49 | if isFatalLevel { 50 | logger.Error("fatal log level is unacceptable, fallbacks to info level") 51 | } 52 | if readFormatError != nil { 53 | logger.Error("failed to read log format from env, fallbacks to json") 54 | } 55 | 56 | logger = logger.WithAndSkip( 57 | 1, 58 | zap.String("commit", meta.LastCommit), 59 | zap.String("version", meta.Version), 60 | zap.String("env", meta.Env), 61 | ) 62 | 63 | return logger, nil 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /internal/meta/meta.go: -------------------------------------------------------------------------------- 1 | package meta 2 | 3 | var ( 4 | Version = "1.0.0" 5 | LastCommit = "abcdefgh" 6 | Env = "dev" 7 | ) 8 | 9 | type Meta struct { 10 | Namespace string `json:"namespace" yaml:"namespace"` 11 | App string `json:"app" yaml:"app"` 12 | Version string `json:"version" yaml:"version"` 13 | LastCommit string `json:"last_commit" yaml:"last_commit"` 14 | Env string `json:"env" yaml:"env"` 15 | } 16 | 17 | func NewMeta(namespace, app string) *Meta { 18 | return &Meta{ 19 | Namespace: namespace, 20 | App: app, 21 | Version: Version, 22 | LastCommit: LastCommit, 23 | Env: Env, 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /internal/types/redis/rediskeys/keys.go: -------------------------------------------------------------------------------- 1 | package rediskeys 2 | 3 | import "fmt" 4 | 5 | // Key key. 6 | type Key string 7 | 8 | // Format format. 9 | func (k Key) Format(params ...interface{}) string { 10 | return fmt.Sprintf(string(k), params...) 11 | } 12 | -------------------------------------------------------------------------------- /pkg/apierrors/apierrors.go: -------------------------------------------------------------------------------- 1 | package apierrors 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "net/http" 7 | "runtime" 8 | 9 | "github.com/bufbuild/protovalidate-go" 10 | "github.com/labstack/echo/v4" 11 | "github.com/lingticio/llmg/apis/jsonapi" 12 | "github.com/samber/lo" 13 | "google.golang.org/grpc/codes" 14 | "google.golang.org/grpc/status" 15 | "google.golang.org/protobuf/runtime/protoiface" 16 | ) 17 | 18 | type Error struct { 19 | *jsonapi.ErrorObject 20 | 21 | caller *jsonapi.ErrorCaller 22 | 23 | grpcStatus uint64 24 | rawError error 25 | } 26 | 27 | func (e *Error) AsStatus() error { 28 | newStatus := status.New(codes.Code(e.grpcStatus), lo.Ternary(e.Detail == "", e.Title, e.Detail)) //nolint:gosec 29 | 30 | details := []protoiface.MessageV1{e.ErrorObject} 31 | if e.Caller() != nil { 32 | details = append(details, e.Caller()) 33 | } 34 | 35 | newStatus, _ = newStatus.WithDetails(details...) 36 | 37 | return newStatus.Err() 38 | } 39 | 40 | func (e *Error) AsResponse() *ErrResponse { 41 | return NewErrResponse().WithError(e) 42 | } 43 | 44 | func (e *Error) AsEchoResponse(c echo.Context) error { 45 | resp := e.AsResponse() 46 | return c.JSON(resp.HTTPStatus(), resp) 47 | } 48 | 49 | func (e *Error) Caller() *jsonapi.ErrorCaller { 50 | return e.caller 51 | } 52 | 53 | func NewError[S ~int, GS ~uint32](status S, grpcStatus GS, code string) *Error { 54 | return &Error{ 55 | ErrorObject: &jsonapi.ErrorObject{ 56 | Id: code, 57 | Status: uint64(status), 58 | Code: code, 59 | }, 60 | grpcStatus: uint64(grpcStatus), 61 | } 62 | } 63 | 64 | func (e *Error) WithError(err error) *Error { 65 | e.rawError = err 66 | e.Detail = err.Error() 67 | 68 | return e 69 | } 70 | 71 | func (e *Error) WithValidationError(err error) *Error { 72 | var validationErr *protovalidate.ValidationError 73 | if !errors.As(err, &validationErr) { 74 | return e.WithDetail(err.Error()) 75 | } 76 | 77 | validationErrProto := validationErr.ToProto() 78 | if len(validationErrProto.GetViolations()) == 0 { 79 | return e.WithDetail(err.Error()) 80 | } 81 | 82 | fieldPath := validationErrProto.GetViolations()[0].GetFieldPath() 83 | forKey := validationErrProto.GetViolations()[0].GetForKey() 84 | message := validationErrProto.GetViolations()[0].GetMessage() 85 | 86 | if forKey { 87 | e.WithDetail(message).WithSourceParameter(fieldPath) 88 | } else { 89 | e.WithDetail(message).WithSourcePointer(fieldPath) 90 | } 91 | 92 | return e 93 | } 94 | 95 | func (e *Error) WithCaller() *Error { 96 | pc, file, line, _ := runtime.Caller(1) 97 | 98 | e.caller = &jsonapi.ErrorCaller{ 99 | Function: runtime.FuncForPC(pc).Name(), 100 | File: file, 101 | Line: int64(line), 102 | } 103 | 104 | return e 105 | } 106 | 107 | func (e *Error) WithTitle(title string) *Error { 108 | e.Title = title 109 | 110 | return e 111 | } 112 | 113 | func (e *Error) WithDetail(detail string) *Error { 114 | e.Detail = detail 115 | 116 | return e 117 | } 118 | 119 | func (e *Error) WithDetailf(format string, args ...interface{}) *Error { 120 | e.Detail = fmt.Sprintf(format, args...) 121 | 122 | return e 123 | } 124 | 125 | func (e *Error) WithSourcePointer(pointer string) *Error { 126 | e.Source = &jsonapi.ErrorObjectSource{ 127 | Pointer: pointer, 128 | } 129 | 130 | return e 131 | } 132 | 133 | func (e *Error) WithSourceParameter(parameter string) *Error { 134 | e.Source = &jsonapi.ErrorObjectSource{ 135 | Parameter: parameter, 136 | } 137 | 138 | return e 139 | } 140 | 141 | func (e *Error) WithSourceHeader(header string) *Error { 142 | e.Source = &jsonapi.ErrorObjectSource{ 143 | Header: header, 144 | } 145 | 146 | return e 147 | } 148 | 149 | type ErrResponse struct { 150 | jsonapi.Response 151 | } 152 | 153 | func NewErrResponseFromErrorObjects(errs ...*jsonapi.ErrorObject) *ErrResponse { 154 | resp := NewErrResponse() 155 | 156 | for _, err := range errs { 157 | resp = resp.WithError(&Error{ 158 | ErrorObject: err, 159 | }) 160 | } 161 | 162 | return resp 163 | } 164 | 165 | func NewErrResponseFromErrorObject(err *jsonapi.ErrorObject) *ErrResponse { 166 | return NewErrResponse().WithError(&Error{ 167 | ErrorObject: err, 168 | }) 169 | } 170 | 171 | func NewErrResponse() *ErrResponse { 172 | return &ErrResponse{ 173 | Response: jsonapi.Response{ 174 | Errors: make([]*jsonapi.ErrorObject, 0), 175 | }, 176 | } 177 | } 178 | 179 | func (e *ErrResponse) WithError(err *Error) *ErrResponse { 180 | e.Errors = append(e.Errors, err.ErrorObject) 181 | 182 | return e 183 | } 184 | 185 | func (e *ErrResponse) WithValidationError(err error) *ErrResponse { 186 | var validationErr *protovalidate.ValidationError 187 | if !errors.As(err, &validationErr) { 188 | return e.WithError(NewErrInvalidArgument().WithError(err)) 189 | } 190 | 191 | validationErrProto := validationErr.ToProto() 192 | if len(validationErrProto.GetViolations()) == 0 { 193 | return e.WithError(NewErrInvalidArgument().WithError(err)) 194 | } 195 | 196 | for _, violation := range validationErrProto.GetViolations() { 197 | fieldPath := violation.GetFieldPath() 198 | forKey := violation.GetForKey() 199 | message := violation.GetMessage() 200 | 201 | if forKey { 202 | e.WithError(NewErrInvalidArgument().WithDetail(message).WithSourceParameter(fieldPath)) 203 | } else { 204 | e.WithError(NewErrInvalidArgument().WithDetail(message).WithSourcePointer(fieldPath)) 205 | } 206 | } 207 | 208 | return e 209 | } 210 | 211 | func (e *ErrResponse) HTTPStatus() int { 212 | if len(e.Errors) == 0 { 213 | return http.StatusOK 214 | } 215 | 216 | return int(e.Errors[0].GetStatus()) //nolint:gosec 217 | } 218 | -------------------------------------------------------------------------------- /pkg/apierrors/errors.go: -------------------------------------------------------------------------------- 1 | package apierrors 2 | 3 | import ( 4 | "net/http" 5 | 6 | "google.golang.org/grpc/codes" 7 | ) 8 | 9 | func NewBadRequest() *Error { 10 | return NewError(http.StatusBadRequest, codes.InvalidArgument, "BAD_REQUEST"). 11 | WithTitle("Bad Request"). 12 | WithDetail("The request was invalid or cannot be served") 13 | } 14 | 15 | func NewErrInternal() *Error { 16 | return NewError(http.StatusInternalServerError, codes.Internal, "INTERNAL_SERVER_ERROR"). 17 | WithTitle("Internal Server Error"). 18 | WithDetail("An internal server error occurred") 19 | } 20 | 21 | func NewPermissionDenied() *Error { 22 | return NewError(http.StatusForbidden, codes.PermissionDenied, "PERMISSION_DENIED"). 23 | WithTitle("Permission Denied"). 24 | WithDetail("You do not have permission to access the requested resources") 25 | } 26 | 27 | func NewErrUnavailable() *Error { 28 | return NewError(http.StatusServiceUnavailable, codes.Unavailable, "UNAVAILABLE"). 29 | WithTitle("Service Unavailable"). 30 | WithDetail("The requested service is unavailable") 31 | } 32 | 33 | func NewErrInvalidArgument() *Error { 34 | return NewError(http.StatusBadRequest, codes.InvalidArgument, "INVALID_ARGUMENT"). 35 | WithTitle("Invalid Argument"). 36 | WithDetail("Invalid parameters, queries, body, or headers were sent, please check the request") 37 | } 38 | 39 | func NewErrUnauthorized() *Error { 40 | return NewError(http.StatusUnauthorized, codes.Unauthenticated, "UNAUTHORIZED"). 41 | WithTitle("Unauthorized"). 42 | WithDetail("The requested resources require authentication") 43 | } 44 | 45 | func NewErrNotFound() *Error { 46 | return NewError(http.StatusNotFound, codes.NotFound, "NOT_FOUND"). 47 | WithTitle("Not Found"). 48 | WithDetail("The requested resources were not found") 49 | } 50 | 51 | func NewPaymentRequired() *Error { 52 | return NewError(http.StatusPaymentRequired, codes.FailedPrecondition, "PAYMENT_REQUIRED"). 53 | WithTitle("Payment Required"). 54 | WithDetail("The requested resources require payment") 55 | } 56 | 57 | func NewQuotaExceeded() *Error { 58 | return NewError(http.StatusTooManyRequests, codes.ResourceExhausted, "QUOTA_EXCEEDED"). 59 | WithTitle("Quota Exceeded"). 60 | WithDetail("The request quota has been exceeded") 61 | } 62 | 63 | func NewForbidden() *Error { 64 | return NewError(http.StatusForbidden, codes.PermissionDenied, "FORBIDDEN"). 65 | WithTitle("Forbidden"). 66 | WithDetail("You do not have permission to access the requested resources") 67 | } 68 | -------------------------------------------------------------------------------- /pkg/configs/cfgproviders/auth.go: -------------------------------------------------------------------------------- 1 | package authstorage 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/lingticio/llmg/pkg/types/metadata" 7 | ) 8 | 9 | type Endpoint struct { 10 | metadata.UnimplementedMetadata 11 | 12 | Tenant metadata.Tenant `json:"tenant" yaml:"tenant"` 13 | Team metadata.Team `json:"team" yaml:"team"` 14 | Group metadata.Group `json:"group" yaml:"group"` 15 | Upstream *metadata.UpstreamSingleOrMultiple `json:"upstream,omitempty" yaml:"upstream,omitempty"` 16 | 17 | ID string `json:"id" yaml:"id"` 18 | Alias string `json:"alias" yaml:"alias"` 19 | APIKey string `json:"apiKey"` 20 | } 21 | 22 | type EndpointProviderQueryable interface { 23 | FindOneByAPIKey(ctx context.Context, apiKey string) (*Endpoint, error) 24 | FindOneByAlias(ctx context.Context, alias string) (*Endpoint, error) 25 | } 26 | 27 | type EndpointProviderMutable interface { 28 | ConfigureOneUpstreamForTenant(ctx context.Context, tenantID string, upstream *metadata.UpstreamSingleOrMultiple) error 29 | ConfigureOneUpstreamForTeam(ctx context.Context, teamID string, upstream *metadata.UpstreamSingleOrMultiple) error 30 | ConfigureOneUpstreamForGroup(ctx context.Context, groupID string, upstream *metadata.UpstreamSingleOrMultiple) error 31 | ConfigureOneUpstreamForEndpoint(ctx context.Context, endpointID string, upstream *metadata.UpstreamSingleOrMultiple) error 32 | ConfigureOne(ctx context.Context, apiKey string, alias string, endpoint *Endpoint) error 33 | } 34 | 35 | type EndpointProvider interface { 36 | EndpointProviderQueryable 37 | } 38 | -------------------------------------------------------------------------------- /pkg/configs/cfgproviders/rds_auth.go: -------------------------------------------------------------------------------- 1 | package authstorage 2 | 3 | import "context" 4 | 5 | var _ EndpointProvider = (*RDSEndpointAuthProvider)(nil) 6 | 7 | type RDSEndpointAuthProvider struct { 8 | } 9 | 10 | func (s *RDSEndpointAuthProvider) FindOneByAPIKey(ctx context.Context, apiKey string) (*Endpoint, error) { 11 | return &Endpoint{}, nil 12 | } 13 | 14 | func (s *RDSEndpointAuthProvider) FindOneByAlias(ctx context.Context, alias string) (*Endpoint, error) { 15 | return &Endpoint{}, nil 16 | } 17 | -------------------------------------------------------------------------------- /pkg/configs/cfgproviders/rds_auth_test.go: -------------------------------------------------------------------------------- 1 | package authstorage 2 | -------------------------------------------------------------------------------- /pkg/configs/cfgproviders/redis_auth.go: -------------------------------------------------------------------------------- 1 | package authstorage 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | 7 | "github.com/lingticio/llmg/pkg/types/metadata" 8 | "github.com/lingticio/llmg/pkg/types/redis/rediskeys" 9 | "github.com/redis/rueidis" 10 | ) 11 | 12 | var _ EndpointProvider = (*RedisEndpointProvider)(nil) 13 | 14 | type RedisEndpointProvider struct { 15 | rueidis rueidis.Client 16 | } 17 | 18 | func NewRedisEndpointAuthProvider() func(rueidis.Client) EndpointProvider { 19 | return func(r rueidis.Client) EndpointProvider { 20 | return &RedisEndpointProvider{ 21 | rueidis: r, 22 | } 23 | } 24 | } 25 | 26 | func (s *RedisEndpointProvider) findUpstreamByRoutesOrGroupID(ctx context.Context, key string) (*metadata.UpstreamSingleOrMultiple, error) { 27 | cmd := s.rueidis.B(). 28 | Get(). 29 | Key(key). 30 | Build() 31 | 32 | res, err := s.rueidis.Do(ctx, cmd).ToString() 33 | if err != nil { 34 | if rueidis.IsRedisNil(err) { 35 | return nil, nil 36 | } 37 | 38 | return nil, err 39 | } 40 | 41 | var upstream metadata.UpstreamSingleOrMultiple 42 | 43 | err = json.Unmarshal([]byte(res), &upstream) 44 | if err != nil { 45 | return nil, err 46 | } 47 | 48 | return &upstream, nil 49 | } 50 | 51 | func (s *RedisEndpointProvider) findUpstreamFromEndpointMetadata(ctx context.Context, endpointMetadata Endpoint) (*metadata.UpstreamSingleOrMultiple, error) { 52 | pipes := []func(ctx context.Context) (*metadata.UpstreamSingleOrMultiple, error){ 53 | func(ctx context.Context) (*metadata.UpstreamSingleOrMultiple, error) { 54 | if endpointMetadata.ID == "" { 55 | return nil, nil 56 | } 57 | 58 | return s.findUpstreamByRoutesOrGroupID(ctx, rediskeys.EndpointUpstreamByEndpointID1.Format(endpointMetadata.ID)) 59 | }, 60 | func(ctx context.Context) (*metadata.UpstreamSingleOrMultiple, error) { 61 | if endpointMetadata.Group.ID() == "" { 62 | return nil, nil 63 | } 64 | 65 | return s.findUpstreamByRoutesOrGroupID(ctx, rediskeys.EndpointUpstreamByGroupID1.Format(endpointMetadata.Group.ID())) 66 | }, 67 | func(ctx context.Context) (*metadata.UpstreamSingleOrMultiple, error) { 68 | if endpointMetadata.Team.ID() == "" { 69 | return nil, nil 70 | } 71 | 72 | return s.findUpstreamByRoutesOrGroupID(ctx, rediskeys.EndpointUpstreamByTeamID1.Format(endpointMetadata.Team.ID())) 73 | }, 74 | func(ctx context.Context) (*metadata.UpstreamSingleOrMultiple, error) { 75 | if endpointMetadata.Tenant.ID() == "" { 76 | return nil, nil 77 | } 78 | 79 | return s.findUpstreamByRoutesOrGroupID(ctx, rediskeys.EndpointUpstreamByTenantID1.Format(endpointMetadata.Tenant.ID())) 80 | }, 81 | } 82 | 83 | var upstream *metadata.UpstreamSingleOrMultiple 84 | 85 | for _, pipe := range pipes { 86 | up, err := pipe(ctx) 87 | if err != nil { 88 | return nil, err 89 | } 90 | if up != nil { 91 | upstream = up 92 | break 93 | } 94 | } 95 | 96 | if upstream == nil { 97 | return nil, nil 98 | } 99 | 100 | return upstream, nil 101 | } 102 | 103 | func (s *RedisEndpointProvider) ConfigureOneUpstreamForTenant(ctx context.Context, tenantID string, upstream *metadata.UpstreamSingleOrMultiple) error { 104 | endpointUpstreamBytes, err := json.Marshal(upstream) 105 | if err != nil { 106 | return err 107 | } 108 | 109 | cmd := s.rueidis.B(). 110 | Set(). 111 | Key(rediskeys.EndpointUpstreamByTenantID1.Format(tenantID)). 112 | Value(string(endpointUpstreamBytes)). 113 | Build() 114 | 115 | err = s.rueidis.Do(ctx, cmd).Error() 116 | if err != nil { 117 | return err 118 | } 119 | 120 | return nil 121 | } 122 | 123 | func (s *RedisEndpointProvider) ConfigureOneUpstreamForTeam(ctx context.Context, teamID string, upstream *metadata.UpstreamSingleOrMultiple) error { 124 | endpointUpstreamBytes, err := json.Marshal(upstream) 125 | if err != nil { 126 | return err 127 | } 128 | 129 | cmd := s.rueidis.B(). 130 | Set(). 131 | Key(rediskeys.EndpointUpstreamByTeamID1.Format(teamID)). 132 | Value(string(endpointUpstreamBytes)). 133 | Build() 134 | 135 | err = s.rueidis.Do(ctx, cmd).Error() 136 | if err != nil { 137 | return err 138 | } 139 | 140 | return nil 141 | } 142 | 143 | func (s *RedisEndpointProvider) ConfigureOneUpstreamForGroup(ctx context.Context, groupID string, upstream *metadata.UpstreamSingleOrMultiple) error { 144 | endpointUpstreamBytes, err := json.Marshal(upstream) 145 | if err != nil { 146 | return err 147 | } 148 | 149 | cmd := s.rueidis.B(). 150 | Set(). 151 | Key(rediskeys.EndpointUpstreamByGroupID1.Format(groupID)). 152 | Value(string(endpointUpstreamBytes)). 153 | Build() 154 | 155 | err = s.rueidis.Do(ctx, cmd).Error() 156 | if err != nil { 157 | return err 158 | } 159 | 160 | return nil 161 | } 162 | 163 | func (s *RedisEndpointProvider) ConfigureOneUpstreamForEndpoint(ctx context.Context, endpointID string, upstream *metadata.UpstreamSingleOrMultiple) error { 164 | endpointUpstreamBytes, err := json.Marshal(upstream) 165 | if err != nil { 166 | return err 167 | } 168 | 169 | cmd := s.rueidis.B(). 170 | Set(). 171 | Key(rediskeys.EndpointUpstreamByEndpointID1.Format(endpointID)). 172 | Value(string(endpointUpstreamBytes)). 173 | Build() 174 | 175 | err = s.rueidis.Do(ctx, cmd).Error() 176 | if err != nil { 177 | return err 178 | } 179 | 180 | return nil 181 | } 182 | 183 | func (s *RedisEndpointProvider) ConfigureOne(ctx context.Context, apiKey string, alias string, endpoint *Endpoint) error { 184 | endpointMetadataBytes, err := json.Marshal(endpoint) 185 | if err != nil { 186 | return err 187 | } 188 | 189 | cmd := s.rueidis.B(). 190 | Set(). 191 | Key(rediskeys.EndpointMetadataByAPIKey1.Format(apiKey)). 192 | Value(string(endpointMetadataBytes)). 193 | Build() 194 | 195 | err = s.rueidis.Do(ctx, cmd).Error() 196 | if err != nil { 197 | return err 198 | } 199 | 200 | cmd = s.rueidis.B(). 201 | Set(). 202 | Key(rediskeys.EndpointMetadataByAlias1.Format(alias)). 203 | Value(string(endpointMetadataBytes)). 204 | Build() 205 | 206 | err = s.rueidis.Do(ctx, cmd).Error() 207 | if err != nil { 208 | return err 209 | } 210 | 211 | return nil 212 | } 213 | 214 | func (s *RedisEndpointProvider) FindOneByAPIKey(ctx context.Context, apiKey string) (*Endpoint, error) { 215 | cmd := s.rueidis.B(). 216 | Get(). 217 | Key(rediskeys.EndpointMetadataByAPIKey1.Format(apiKey)). 218 | Build() 219 | 220 | res, err := s.rueidis.Do(ctx, cmd).ToString() 221 | if err != nil { 222 | if rueidis.IsRedisNil(err) { 223 | return nil, nil 224 | } 225 | 226 | return nil, err 227 | } 228 | 229 | var endpointMetadata Endpoint 230 | 231 | err = json.Unmarshal([]byte(res), &endpointMetadata) 232 | if err != nil { 233 | return nil, err 234 | } 235 | 236 | upstream, err := s.findUpstreamFromEndpointMetadata(ctx, endpointMetadata) 237 | if err != nil { 238 | return nil, err 239 | } 240 | if upstream == nil { 241 | return nil, nil 242 | } 243 | 244 | return &Endpoint{ 245 | Tenant: endpointMetadata.Tenant, 246 | Team: endpointMetadata.Team, 247 | Group: endpointMetadata.Group, 248 | Upstream: upstream, 249 | ID: endpointMetadata.ID, 250 | Alias: endpointMetadata.Alias, 251 | APIKey: endpointMetadata.APIKey, 252 | }, nil 253 | } 254 | 255 | func (s *RedisEndpointProvider) FindOneByAlias(ctx context.Context, alias string) (*Endpoint, error) { 256 | cmd := s.rueidis.B(). 257 | Get(). 258 | Key(rediskeys.EndpointMetadataByAlias1.Format(alias)). 259 | Build() 260 | 261 | res, err := s.rueidis.Do(ctx, cmd).ToString() 262 | if err != nil { 263 | if rueidis.IsRedisNil(err) { 264 | return nil, nil 265 | } 266 | 267 | return nil, err 268 | } 269 | 270 | var endpointMetadata Endpoint 271 | 272 | err = json.Unmarshal([]byte(res), &endpointMetadata) 273 | if err != nil { 274 | return nil, err 275 | } 276 | 277 | upstream, err := s.findUpstreamFromEndpointMetadata(ctx, endpointMetadata) 278 | if err != nil { 279 | return nil, err 280 | } 281 | if upstream == nil { 282 | return nil, nil 283 | } 284 | 285 | return &Endpoint{ 286 | Tenant: endpointMetadata.Tenant, 287 | Team: endpointMetadata.Team, 288 | Group: endpointMetadata.Group, 289 | Upstream: upstream, 290 | ID: endpointMetadata.ID, 291 | Alias: endpointMetadata.Alias, 292 | APIKey: endpointMetadata.APIKey, 293 | }, nil 294 | } 295 | -------------------------------------------------------------------------------- /pkg/configs/cfgproviders/redis_auth_test.go: -------------------------------------------------------------------------------- 1 | package authstorage 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/lingticio/llmg/pkg/types/metadata" 8 | "github.com/redis/rueidis" 9 | "github.com/stretchr/testify/assert" 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | func TestRedisEndpointProvider_FindOneByAPIKey(t *testing.T) { 14 | r, err := rueidis.NewClient(rueidis.ClientOption{ 15 | InitAddress: []string{"localhost:6379"}, 16 | }) 17 | require.NoError(t, err) 18 | require.NotNil(t, r) 19 | 20 | defer r.Close() 21 | 22 | rp := NewRedisEndpointAuthProvider()(r) 23 | redisProvider, ok := rp.(*RedisEndpointProvider) 24 | require.True(t, ok) 25 | require.NotNil(t, redisProvider) 26 | 27 | err = redisProvider.ConfigureOneUpstreamForEndpoint(context.Background(), "endpointId", &metadata.UpstreamSingleOrMultiple{ 28 | Upstream: &metadata.Upstream{ 29 | OpenAI: metadata.UpstreamOpenAI{ 30 | BaseURL: "baseURL", 31 | APIKey: "apiKey", 32 | }, 33 | }, 34 | }) 35 | require.NoError(t, err) 36 | 37 | err = redisProvider.ConfigureOne(context.Background(), "apiKey", "alias", &Endpoint{ 38 | Tenant: metadata.Tenant{Id: "tenantId"}, 39 | Team: metadata.Team{Id: "teamId"}, 40 | Group: metadata.Group{Id: "groupId"}, 41 | ID: "endpointId", 42 | Alias: "alias", 43 | APIKey: "apiKey", 44 | }) 45 | require.NoError(t, err) 46 | 47 | endpoint, err := rp.FindOneByAPIKey(context.Background(), "apiKey") 48 | require.NoError(t, err) 49 | 50 | assert.Equal(t, "endpointId", endpoint.ID) 51 | assert.Equal(t, "alias", endpoint.Alias) 52 | assert.Equal(t, "apiKey", endpoint.APIKey) 53 | assert.Equal(t, "tenantId", endpoint.Tenant.Id) 54 | assert.Equal(t, "teamId", endpoint.Team.Id) 55 | assert.Equal(t, "groupId", endpoint.Group.Id) 56 | assert.Equal(t, "baseURL", endpoint.Upstream.OpenAI.BaseURL) 57 | assert.Equal(t, "apiKey", endpoint.Upstream.OpenAI.APIKey) 58 | } 59 | 60 | func TestRedisEndpointProvider_FindOneByAlias(t *testing.T) { 61 | r, err := rueidis.NewClient(rueidis.ClientOption{ 62 | InitAddress: []string{"localhost:6379"}, 63 | }) 64 | require.NoError(t, err) 65 | require.NotNil(t, r) 66 | 67 | defer r.Close() 68 | 69 | rp := NewRedisEndpointAuthProvider()(r) 70 | redisProvider, ok := rp.(*RedisEndpointProvider) 71 | require.True(t, ok) 72 | require.NotNil(t, redisProvider) 73 | 74 | err = redisProvider.ConfigureOneUpstreamForEndpoint(context.Background(), "endpointId", &metadata.UpstreamSingleOrMultiple{ 75 | Upstream: &metadata.Upstream{ 76 | OpenAI: metadata.UpstreamOpenAI{ 77 | BaseURL: "baseURL", 78 | APIKey: "apiKey", 79 | }, 80 | }, 81 | }) 82 | require.NoError(t, err) 83 | 84 | err = redisProvider.ConfigureOne(context.Background(), "apiKey", "alias", &Endpoint{ 85 | Tenant: metadata.Tenant{Id: "tenantId"}, 86 | Team: metadata.Team{Id: "teamId"}, 87 | Group: metadata.Group{Id: "groupId"}, 88 | ID: "endpointId", 89 | Alias: "alias", 90 | APIKey: "apiKey", 91 | }) 92 | require.NoError(t, err) 93 | 94 | endpoint, err := rp.FindOneByAlias(context.Background(), "alias") 95 | require.NoError(t, err) 96 | 97 | assert.Equal(t, "endpointId", endpoint.ID) 98 | assert.Equal(t, "alias", endpoint.Alias) 99 | assert.Equal(t, "apiKey", endpoint.APIKey) 100 | assert.Equal(t, "tenantId", endpoint.Tenant.Id) 101 | assert.Equal(t, "teamId", endpoint.Team.Id) 102 | assert.Equal(t, "groupId", endpoint.Group.Id) 103 | assert.Equal(t, "baseURL", endpoint.Upstream.OpenAI.BaseURL) 104 | assert.Equal(t, "apiKey", endpoint.Upstream.OpenAI.APIKey) 105 | } 106 | -------------------------------------------------------------------------------- /pkg/configs/cfgproviders/static_auth.go: -------------------------------------------------------------------------------- 1 | package authstorage 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | 7 | "github.com/lingticio/llmg/internal/configs" 8 | "github.com/lingticio/llmg/pkg/types/metadata" 9 | ) 10 | 11 | var _ EndpointProvider = (*ConfigEndpointProvider)(nil) 12 | 13 | type ConfigEndpointProvider struct { 14 | Config *configs.Routes 15 | } 16 | 17 | func (s *ConfigEndpointProvider) findUpstream(endpoint configs.Endpoint, group configs.Group, team configs.Team, tenant configs.Tenant) *metadata.UpstreamSingleOrMultiple { 18 | if endpoint.Upstream != nil { 19 | return endpoint.Upstream 20 | } 21 | if group.Upstream != nil { 22 | return group.Upstream 23 | } 24 | if team.Upstream != nil { 25 | return team.Upstream 26 | } 27 | 28 | return tenant.Upstream 29 | } 30 | 31 | func (s *ConfigEndpointProvider) searchGroupsForAPIKey(tenantID, teamID string, groups []configs.Group, apiKey string, team configs.Team, tenant configs.Tenant) (*Endpoint, error) { 32 | for _, group := range groups { 33 | // Search in current group's endpoints 34 | for _, endpoint := range group.Endpoints { 35 | if endpoint.APIKey == apiKey { 36 | return &Endpoint{ 37 | Tenant: metadata.Tenant{Id: tenantID}, 38 | Team: metadata.Team{Id: teamID}, 39 | Group: metadata.Group{Id: group.ID}, 40 | ID: endpoint.ID, 41 | Alias: endpoint.Alias, 42 | APIKey: endpoint.APIKey, 43 | Upstream: s.findUpstream(endpoint, group, team, tenant), 44 | }, nil 45 | } 46 | } 47 | 48 | // Recursively search in nested groups 49 | if len(group.Groups) > 0 { 50 | metadata, err := s.searchGroupsForAPIKey(tenantID, teamID, group.Groups, apiKey, team, tenant) 51 | if err == nil { 52 | return metadata, nil 53 | } 54 | } 55 | } 56 | 57 | return nil, errors.New("api key not found") 58 | } 59 | 60 | func (s *ConfigEndpointProvider) FindOneByAPIKey(ctx context.Context, apiKey string) (*Endpoint, error) { 61 | for _, tenant := range s.Config.Tenants { 62 | for _, team := range tenant.Teams { 63 | metadata, err := s.searchGroupsForAPIKey(tenant.ID, team.ID, team.Groups, apiKey, team, tenant) 64 | if err == nil { 65 | return metadata, nil 66 | } 67 | } 68 | } 69 | 70 | return nil, errors.New("api key not found") 71 | } 72 | 73 | func (s *ConfigEndpointProvider) searchGroupsForAlias(tenantID, teamID string, groups []configs.Group, alias string, team configs.Team, tenant configs.Tenant) (*Endpoint, error) { 74 | for _, group := range groups { 75 | // Search in current group's endpoints 76 | for _, endpoint := range group.Endpoints { 77 | if endpoint.Alias == alias { 78 | return &Endpoint{ 79 | Tenant: metadata.Tenant{Id: tenantID}, 80 | Team: metadata.Team{Id: teamID}, 81 | Group: metadata.Group{Id: group.ID}, 82 | ID: endpoint.ID, 83 | Alias: endpoint.Alias, 84 | APIKey: endpoint.APIKey, 85 | Upstream: s.findUpstream(endpoint, group, team, tenant), 86 | }, nil 87 | } 88 | } 89 | 90 | // Recursively search in nested groups 91 | if len(group.Groups) > 0 { 92 | metadata, err := s.searchGroupsForAlias(tenantID, teamID, group.Groups, alias, team, tenant) 93 | if err == nil { 94 | return metadata, nil 95 | } 96 | } 97 | } 98 | 99 | return nil, errors.New("alias not found") 100 | } 101 | 102 | func (s *ConfigEndpointProvider) FindOneByAlias(ctx context.Context, alias string) (*Endpoint, error) { 103 | for _, tenant := range s.Config.Tenants { 104 | for _, team := range tenant.Teams { 105 | metadata, err := s.searchGroupsForAlias(tenant.ID, team.ID, team.Groups, alias, team, tenant) 106 | if err == nil { 107 | return metadata, nil 108 | } 109 | } 110 | } 111 | 112 | return nil, errors.New("alias not found") 113 | } 114 | -------------------------------------------------------------------------------- /pkg/configs/cfgproviders/static_auth_test.go: -------------------------------------------------------------------------------- 1 | package authstorage 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/lingticio/llmg/internal/configs" 8 | "github.com/lingticio/llmg/pkg/types/metadata" 9 | "github.com/nekomeowww/xo" 10 | "github.com/stretchr/testify/assert" 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | func TestConfigEndpointProvider_FindOneByAPIKey(t *testing.T) { 15 | tenantID := xo.RandomHashString(8) 16 | teamID := xo.RandomHashString(8) 17 | groupID1 := xo.RandomHashString(8) 18 | groupID2 := xo.RandomHashString(8) 19 | 20 | apiKey1 := xo.RandomHashString(16) 21 | apiKey2 := xo.RandomHashString(16) 22 | 23 | s := &ConfigEndpointProvider{ 24 | Config: &configs.Routes{ 25 | Tenants: []configs.Tenant{ 26 | { 27 | ID: tenantID, 28 | Teams: []configs.Team{ 29 | { 30 | ID: teamID, 31 | Groups: []configs.Group{ 32 | { 33 | ID: groupID1, 34 | Groups: []configs.Group{ 35 | { 36 | ID: groupID2, 37 | Endpoints: []configs.Endpoint{ 38 | { 39 | ID: xo.RandomHashString(8), 40 | Alias: "test", 41 | APIKey: apiKey1, 42 | }, 43 | }, 44 | }, 45 | }, 46 | Endpoints: []configs.Endpoint{ 47 | { 48 | ID: xo.RandomHashString(8), 49 | Alias: "test-2", 50 | APIKey: apiKey2, 51 | }, 52 | }, 53 | }, 54 | }, 55 | }, 56 | }, 57 | }, 58 | }, 59 | }, 60 | } 61 | 62 | md, err := s.FindOneByAPIKey(context.TODO(), apiKey1) 63 | require.NoError(t, err) 64 | require.NotNil(t, md) 65 | 66 | assert.Equal(t, tenantID, md.Tenant.ID()) 67 | assert.Equal(t, teamID, md.Team.ID()) 68 | assert.Equal(t, groupID2, md.Group.ID()) 69 | assert.Equal(t, apiKey1, md.APIKey) 70 | 71 | md, err = s.FindOneByAPIKey(context.TODO(), apiKey2) 72 | require.NoError(t, err) 73 | require.NotNil(t, md) 74 | 75 | assert.Equal(t, tenantID, md.Tenant.ID()) 76 | assert.Equal(t, teamID, md.Team.ID()) 77 | assert.Equal(t, groupID1, md.Group.ID()) 78 | assert.Equal(t, apiKey2, md.APIKey) 79 | 80 | md, err = s.FindOneByAPIKey(context.TODO(), "invalid") 81 | require.Error(t, err) 82 | require.Nil(t, md) 83 | } 84 | 85 | func TestConfigEndpointProvider_FindOneByAlias(t *testing.T) { 86 | tenantID := xo.RandomHashString(8) 87 | teamID := xo.RandomHashString(8) 88 | groupID1 := xo.RandomHashString(8) 89 | groupID2 := xo.RandomHashString(8) 90 | 91 | alias1 := "test" 92 | alias2 := "test-2" 93 | 94 | s := &ConfigEndpointProvider{ 95 | Config: &configs.Routes{ 96 | Tenants: []configs.Tenant{ 97 | { 98 | ID: tenantID, 99 | Teams: []configs.Team{ 100 | { 101 | ID: teamID, 102 | Groups: []configs.Group{ 103 | { 104 | ID: groupID1, 105 | Groups: []configs.Group{ 106 | { 107 | ID: groupID2, 108 | Endpoints: []configs.Endpoint{ 109 | { 110 | ID: xo.RandomHashString(8), 111 | Alias: alias1, 112 | APIKey: xo.RandomHashString(16), 113 | }, 114 | }, 115 | }, 116 | }, 117 | Endpoints: []configs.Endpoint{ 118 | { 119 | ID: xo.RandomHashString(8), 120 | Alias: alias2, 121 | APIKey: xo.RandomHashString(16), 122 | }, 123 | }, 124 | }, 125 | }, 126 | }, 127 | }, 128 | }, 129 | }, 130 | }, 131 | } 132 | 133 | md, err := s.FindOneByAlias(context.TODO(), alias1) 134 | require.NoError(t, err) 135 | require.NotNil(t, md) 136 | 137 | assert.Equal(t, tenantID, md.Tenant.ID()) 138 | assert.Equal(t, teamID, md.Team.ID()) 139 | assert.Equal(t, groupID2, md.Group.ID()) 140 | assert.Equal(t, alias1, md.Alias) 141 | 142 | md, err = s.FindOneByAlias(context.TODO(), alias2) 143 | require.NoError(t, err) 144 | require.NotNil(t, md) 145 | 146 | assert.Equal(t, tenantID, md.Tenant.ID()) 147 | assert.Equal(t, teamID, md.Team.ID()) 148 | assert.Equal(t, groupID1, md.Group.ID()) 149 | assert.Equal(t, alias2, md.Alias) 150 | 151 | md, err = s.FindOneByAlias(context.TODO(), "invalid") 152 | require.Error(t, err) 153 | require.Nil(t, md) 154 | } 155 | 156 | func TestConfigAuthStorage_findUpstream(t *testing.T) { 157 | tenantID := xo.RandomHashString(8) 158 | teamID := xo.RandomHashString(8) 159 | groupID := xo.RandomHashString(8) 160 | 161 | tenantUpstream := &metadata.UpstreamSingleOrMultiple{ 162 | Upstream: &metadata.Upstream{ 163 | OpenAI: metadata.UpstreamOpenAI{ 164 | BaseURL: "tenant-url", 165 | APIKey: "tenant-key", 166 | }, 167 | }, 168 | } 169 | 170 | teamUpstream := &metadata.UpstreamSingleOrMultiple{ 171 | Upstream: &metadata.Upstream{ 172 | OpenAI: metadata.UpstreamOpenAI{ 173 | BaseURL: "team-url", 174 | APIKey: "team-key", 175 | }, 176 | }, 177 | } 178 | 179 | groupUpstream := &metadata.UpstreamSingleOrMultiple{ 180 | Upstream: &metadata.Upstream{ 181 | OpenAI: metadata.UpstreamOpenAI{ 182 | BaseURL: "group-url", 183 | APIKey: "group-key", 184 | }, 185 | }, 186 | } 187 | 188 | endpointUpstream := &metadata.UpstreamSingleOrMultiple{ 189 | Upstream: &metadata.Upstream{ 190 | OpenAI: metadata.UpstreamOpenAI{ 191 | BaseURL: "endpoint-url", 192 | APIKey: "endpoint-key", 193 | }, 194 | }, 195 | } 196 | 197 | s := &ConfigEndpointProvider{ 198 | Config: &configs.Routes{ 199 | Tenants: []configs.Tenant{ 200 | { 201 | ID: tenantID, 202 | Upstream: tenantUpstream, 203 | Teams: []configs.Team{ 204 | { 205 | ID: teamID, 206 | Upstream: teamUpstream, 207 | Groups: []configs.Group{ 208 | { 209 | ID: groupID, 210 | Upstream: groupUpstream, 211 | Endpoints: []configs.Endpoint{ 212 | { 213 | ID: "endpoint1", 214 | APIKey: "key1", 215 | Upstream: endpointUpstream, 216 | }, 217 | { 218 | ID: "endpoint2", 219 | APIKey: "key2", 220 | // No upstream - should inherit from group 221 | }, 222 | }, 223 | }, 224 | { 225 | ID: "group2", 226 | Endpoints: []configs.Endpoint{ 227 | { 228 | ID: "endpoint3", 229 | APIKey: "key3", 230 | // No upstream - should inherit from team 231 | }, 232 | }, 233 | }, 234 | }, 235 | }, 236 | }, 237 | }, 238 | }, 239 | }, 240 | } 241 | 242 | // Test endpoint with its own upstream 243 | md1, err := s.FindOneByAPIKey(context.TODO(), "key1") 244 | require.NoError(t, err) 245 | assert.Equal(t, endpointUpstream, md1.Upstream) 246 | 247 | // Test endpoint inheriting from group 248 | md2, err := s.FindOneByAPIKey(context.TODO(), "key2") 249 | require.NoError(t, err) 250 | assert.Equal(t, groupUpstream, md2.Upstream) 251 | 252 | // Test endpoint inheriting from team 253 | md3, err := s.FindOneByAPIKey(context.TODO(), "key3") 254 | require.NoError(t, err) 255 | assert.Equal(t, teamUpstream, md3.Upstream) 256 | } 257 | -------------------------------------------------------------------------------- /pkg/neuri/formats/jsonfmt/json_parser.go: -------------------------------------------------------------------------------- 1 | package jsonfmt 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "unicode" 7 | ) 8 | 9 | type TokenType int 10 | 11 | const ( 12 | TokenTypeText TokenType = iota 13 | TokenTypeJSONObject 14 | TokenTypeJSONArray 15 | TokenTypeJSONField 16 | TokenTypeJSONString 17 | TokenTypeJSONNumber 18 | TokenTypeJSONBoolean 19 | TokenTypeJSONNull 20 | ) 21 | 22 | type Pos struct { 23 | Offset int 24 | Line int 25 | Column int 26 | } 27 | 28 | type Token struct { 29 | Type TokenType 30 | Content string 31 | Pos Pos 32 | Children []*Token 33 | } 34 | 35 | func (t *Token) stringifyObject() string { 36 | parts := make([]string, 0, len(t.Children)) 37 | 38 | for _, child := range t.Children { 39 | parts = append(parts, child.String()) 40 | } 41 | 42 | return fmt.Sprintf("{%s}", strings.Join(parts, ",")) 43 | } 44 | 45 | func (t *Token) stringifyArray() string { 46 | parts := make([]string, 0, len(t.Children)) 47 | 48 | for _, child := range t.Children { 49 | parts = append(parts, child.String()) 50 | } 51 | 52 | return fmt.Sprintf("[%s]", strings.Join(parts, ",")) 53 | } 54 | 55 | func (t *Token) stringifyField() string { 56 | if len(t.Children) > 0 { 57 | childValue := t.Children[0].String() 58 | return fmt.Sprintf("\"%s\":%s", t.Content, childValue) 59 | } 60 | 61 | return fmt.Sprintf("\"%s\":null", t.Content) 62 | } 63 | 64 | func (t *Token) String() string { 65 | switch t.Type { 66 | case TokenTypeText: 67 | return t.Content 68 | case TokenTypeJSONObject: 69 | return t.stringifyObject() 70 | case TokenTypeJSONArray: 71 | return t.stringifyArray() 72 | case TokenTypeJSONField: 73 | return t.stringifyField() 74 | case TokenTypeJSONString: 75 | return t.Content 76 | case TokenTypeJSONNumber, TokenTypeJSONBoolean, TokenTypeJSONNull: 77 | return t.Content 78 | default: 79 | return "" 80 | } 81 | } 82 | 83 | type ParserState int 84 | 85 | const ( 86 | ParserStateText ParserState = iota 87 | ParserStateJSONStart 88 | ParserStateJSONString 89 | ParserStateJSONEscape 90 | ParserStateJSONFieldName 91 | ParserStateJSONFieldValue 92 | ParserStateJSONNumber 93 | ) 94 | 95 | type JSONParser struct { 96 | buffer strings.Builder 97 | 98 | // States 99 | inSingleQuote bool 100 | insideJSON bool 101 | state ParserState 102 | stateStack []ParserState 103 | depth int 104 | tokenStart Pos 105 | pos Pos 106 | 107 | // Cursor 108 | currentToken *Token 109 | currentContainer *Token 110 | containerStack []*Token 111 | 112 | // Tree structure 113 | tree []*Token 114 | } 115 | 116 | func NewJSONParser() *JSONParser { 117 | return &JSONParser{ 118 | state: ParserStateText, 119 | stateStack: make([]ParserState, 0), 120 | pos: Pos{Line: 1, Column: 1}, 121 | containerStack: make([]*Token, 0), 122 | tree: make([]*Token, 0), 123 | } 124 | } 125 | 126 | func (p *JSONParser) Parse(chunk string) []*Token { 127 | for _, char := range chunk { 128 | p.processChar(char) 129 | p.updatePosition(char) 130 | } 131 | 132 | return p.getCompletedTokens() 133 | } 134 | 135 | func (p *JSONParser) End() []*Token { 136 | if p.insideJSON { 137 | p.autoCloseJSON() 138 | } 139 | 140 | p.flushBuffer() 141 | p.insideJSON = false 142 | 143 | return p.tree 144 | } 145 | 146 | func TokensToString(tokens []*Token) string { 147 | var result strings.Builder 148 | 149 | for _, token := range tokens { 150 | if token.Type == TokenTypeJSONObject || token.Type == TokenTypeJSONArray { 151 | result.WriteString(token.String()) 152 | } 153 | } 154 | 155 | return result.String() 156 | } 157 | 158 | func (p *JSONParser) pushState(state ParserState) { 159 | p.stateStack = append(p.stateStack, state) 160 | p.state = state 161 | } 162 | 163 | func (p *JSONParser) popState() { 164 | if len(p.stateStack) > 1 { 165 | p.stateStack = p.stateStack[:len(p.stateStack)-1] 166 | p.state = p.stateStack[len(p.stateStack)-1] 167 | } 168 | } 169 | 170 | func (p *JSONParser) handleStateJSONNumber(char rune) { 171 | if unicode.IsDigit(char) || char == '.' || char == 'e' || char == 'E' || char == '+' || char == '-' { 172 | p.buffer.WriteRune(char) 173 | } else { 174 | p.completeCurrentToken() 175 | p.popState() // Return to previous state 176 | p.processChar(char) 177 | } 178 | } 179 | 180 | func (p *JSONParser) completeCurrentToken() { 181 | value := p.buffer.String() 182 | if value == "" { 183 | // Don't create a token for an empty buffer 184 | return 185 | } 186 | 187 | tokenType := p.determineValueType(value) 188 | valueToken := &Token{Type: tokenType, Content: value, Pos: p.tokenStart} 189 | 190 | if len(p.containerStack) > 0 { 191 | currentContainer := p.containerStack[len(p.containerStack)-1] 192 | if p.currentToken != nil && p.currentToken.Type == TokenTypeJSONField { 193 | p.currentToken.Children = append(p.currentToken.Children, valueToken) 194 | } else { 195 | currentContainer.Children = append(currentContainer.Children, valueToken) 196 | } 197 | } else { 198 | p.tree = append(p.tree, valueToken) 199 | } 200 | 201 | p.buffer.Reset() 202 | p.tokenStart = p.pos 203 | } 204 | 205 | func (p *JSONParser) processChar(char rune) { 206 | switch p.state { 207 | case ParserStateText: 208 | p.handleStateText(char) 209 | case ParserStateJSONStart: 210 | p.handleStateJSONStart(char) 211 | case ParserStateJSONString: 212 | p.handleStateJSONString(char) 213 | case ParserStateJSONEscape: 214 | p.handleStateJSONEscape(char) 215 | case ParserStateJSONFieldName: 216 | p.handleStateJSONFieldName(char) 217 | case ParserStateJSONFieldValue: 218 | p.handleStateJSONFieldValue(char) 219 | case ParserStateJSONNumber: 220 | p.handleStateJSONNumber(char) 221 | } 222 | } 223 | 224 | func (p *JSONParser) handleStateText(char rune) { 225 | if char == '{' || char == '[' { 226 | p.flushBuffer() 227 | p.startNewJSONToken(char) 228 | p.insideJSON = true 229 | p.pushState(ParserStateJSONStart) 230 | } else { 231 | if p.buffer.Len() == 0 { 232 | p.tokenStart = p.pos 233 | } 234 | 235 | p.buffer.WriteRune(char) 236 | } 237 | } 238 | 239 | func (p *JSONParser) handleStateJSONStart(char rune) { 240 | switch { 241 | case char == '"' || char == '\'': 242 | p.pushState(ParserStateJSONString) 243 | p.inSingleQuote = (char == '\'') 244 | p.buffer.WriteRune(char) 245 | case char == '}' || char == ']': 246 | p.completeCurrentToken() 247 | p.depth-- 248 | 249 | if len(p.containerStack) > 0 { 250 | p.containerStack = p.containerStack[:len(p.containerStack)-1] 251 | } 252 | 253 | if p.depth == 0 { 254 | p.insideJSON = false 255 | p.popState() // Should return to StateText 256 | p.flushBuffer() 257 | p.currentContainer = nil 258 | } else if len(p.containerStack) > 0 { 259 | p.currentContainer = p.containerStack[len(p.containerStack)-1] 260 | } 261 | p.currentToken = p.currentContainer 262 | case char == '{' || char == '[': 263 | p.completeCurrentToken() 264 | p.startNewJSONToken(char) 265 | case char == ':': 266 | p.startNewJSONField() 267 | p.pushState(ParserStateJSONFieldValue) 268 | case char == ',': 269 | p.completeCurrentToken() 270 | if p.currentContainer != nil && p.currentContainer.Type == TokenTypeJSONArray { 271 | p.currentToken = p.currentContainer 272 | } 273 | case unicode.IsDigit(char) || char == '-': 274 | p.pushState(ParserStateJSONNumber) 275 | p.buffer.WriteRune(char) 276 | case char == 't' || char == 'f' || char == 'n': 277 | p.pushState(ParserStateJSONFieldValue) 278 | p.buffer.WriteRune(char) 279 | default: 280 | if !unicode.IsSpace(char) { 281 | if p.currentContainer != nil && p.currentContainer.Type == TokenTypeJSONArray { 282 | p.startNewJSONToken('{') 283 | p.pushState(ParserStateJSONFieldName) 284 | } else { 285 | p.pushState(ParserStateJSONFieldName) 286 | } 287 | 288 | p.buffer.WriteRune(char) 289 | } 290 | } 291 | } 292 | 293 | func (p *JSONParser) handleStateJSONString(char rune) { 294 | p.buffer.WriteRune(char) 295 | if char == '\\' { 296 | p.pushState(ParserStateJSONEscape) 297 | } else if (char == '"' && !p.inSingleQuote) || (char == '\'' && p.inSingleQuote) { 298 | p.popState() 299 | p.inSingleQuote = false 300 | } 301 | } 302 | 303 | func (p *JSONParser) handleStateJSONEscape(char rune) { 304 | p.buffer.WriteRune(char) 305 | p.popState() // Return to StateJSONString 306 | } 307 | 308 | func (p *JSONParser) handleStateJSONFieldName(char rune) { 309 | if char == ':' { 310 | p.startNewJSONField() 311 | p.popState() 312 | p.pushState(ParserStateJSONFieldValue) 313 | } else { 314 | p.buffer.WriteRune(char) 315 | } 316 | } 317 | 318 | func (p *JSONParser) handleStateJSONFieldValue(char rune) { 319 | switch { 320 | case char == ',' || char == '}' || char == ']': 321 | p.completeCurrentToken() 322 | p.popState() // Return to StateJSONStart 323 | if char == '}' || char == ']' { 324 | p.processChar(char) 325 | } 326 | case char == '{' || char == '[': 327 | p.startNewJSONToken(char) 328 | case char == '"' || char == '\'': 329 | p.pushState(ParserStateJSONString) 330 | p.inSingleQuote = (char == '\'') 331 | p.buffer.WriteRune(char) 332 | case unicode.IsDigit(char) || char == '-': 333 | p.popState() 334 | p.pushState(ParserStateJSONNumber) 335 | p.buffer.WriteRune(char) 336 | case char == 't' || char == 'f' || char == 'n': 337 | p.buffer.WriteRune(char) 338 | default: 339 | if !unicode.IsSpace(char) { 340 | p.buffer.WriteRune(char) 341 | } 342 | } 343 | } 344 | 345 | func (p *JSONParser) determineValueType(value string) TokenType { 346 | switch { 347 | case value == "true" || value == "false": 348 | return TokenTypeJSONBoolean 349 | case value == "null": 350 | return TokenTypeJSONNull 351 | case len(value) > 0 && (unicode.IsDigit(rune(value[0])) || value[0] == '-'): 352 | return TokenTypeJSONNumber 353 | default: 354 | return TokenTypeJSONString 355 | } 356 | } 357 | 358 | func (p *JSONParser) startNewJSONToken(char rune) { 359 | var tokenType TokenType 360 | if char == '{' { 361 | tokenType = TokenTypeJSONObject 362 | } else { 363 | tokenType = TokenTypeJSONArray 364 | } 365 | 366 | newToken := &Token{Type: tokenType, Pos: p.pos, Children: make([]*Token, 0)} 367 | 368 | if len(p.containerStack) > 0 { 369 | currentContainer := p.containerStack[len(p.containerStack)-1] 370 | if p.currentToken != nil && p.currentToken.Type == TokenTypeJSONField { 371 | p.currentToken.Children = append(p.currentToken.Children, newToken) 372 | } else { 373 | currentContainer.Children = append(currentContainer.Children, newToken) 374 | } 375 | } else { 376 | p.tree = append(p.tree, newToken) 377 | } 378 | 379 | p.containerStack = append(p.containerStack, newToken) 380 | p.currentContainer = newToken 381 | p.currentToken = newToken 382 | p.pushState(ParserStateJSONStart) 383 | p.depth++ 384 | } 385 | 386 | func (p *JSONParser) startNewJSONField() { 387 | fieldName := strings.TrimSpace(p.buffer.String()) 388 | fieldName = strings.Trim(fieldName, "\"'") 389 | newToken := &Token{Type: TokenTypeJSONField, Content: fieldName, Pos: p.tokenStart, Children: make([]*Token, 0)} 390 | p.currentContainer.Children = append(p.currentContainer.Children, newToken) 391 | p.currentToken = newToken 392 | p.buffer.Reset() 393 | } 394 | 395 | func (p *JSONParser) autoCloseJSON() { 396 | for p.depth > 0 { 397 | p.depth-- 398 | p.completeCurrentToken() 399 | } 400 | } 401 | 402 | func (p *JSONParser) flushBuffer() { 403 | if p.buffer.Len() > 0 { 404 | content := p.buffer.String() 405 | if !p.insideJSON { 406 | p.tree = append(p.tree, &Token{Type: TokenTypeText, Content: content, Pos: p.tokenStart}) 407 | } 408 | 409 | p.buffer.Reset() 410 | } 411 | } 412 | 413 | func (p *JSONParser) getCompletedTokens() []*Token { 414 | var completedTokens []*Token 415 | 416 | for _, token := range p.tree { 417 | if token.Type == TokenTypeText || (token.Type == TokenTypeJSONObject || token.Type == TokenTypeJSONArray) && p.state == ParserStateText { 418 | completedTokens = append(completedTokens, token) 419 | } else { 420 | break 421 | } 422 | } 423 | 424 | p.tree = p.tree[len(completedTokens):] 425 | 426 | return completedTokens 427 | } 428 | 429 | func (p *JSONParser) updatePosition(char rune) { 430 | p.pos.Offset++ 431 | if char == '\n' { 432 | p.pos.Line++ 433 | p.pos.Column = 1 434 | } else { 435 | p.pos.Column++ 436 | } 437 | } 438 | -------------------------------------------------------------------------------- /pkg/neuri/formats/jsonfmt/json_parser_test.go: -------------------------------------------------------------------------------- 1 | package jsonfmt 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestNewJSONParser(t *testing.T) { 12 | parser := NewJSONParser() 13 | testStr := "Some of the test string\n```json\n{\"name\": \"abcd\",\n\"age\": 30\n}```" 14 | allTokens := make([]*Token, 0) 15 | 16 | for _, char := range testStr { 17 | allTokens = append(allTokens, parser.Parse(string(char))...) 18 | } 19 | 20 | allTokens = append(allTokens, parser.End()...) 21 | 22 | printTokenTree(allTokens, 0) 23 | assert.Equal(t, "{\"name\":\"abcd\",\"age\":30}", TokensToString(allTokens), "JSON String is not as expected") 24 | } 25 | 26 | func TestNewJSONParserWithNestedObject(t *testing.T) { 27 | parser := NewJSONParser() 28 | testStr := "Some of the test string\n```json\n{\"name\": \"abcd\",\n\"age\": 30\n,\"address\": {\"city\": \"New York\", \"zip\": 10001}}" 29 | allTokens := make([]*Token, 0) 30 | 31 | for _, char := range testStr { 32 | allTokens = append(allTokens, parser.Parse(string(char))...) 33 | } 34 | 35 | allTokens = append(allTokens, parser.End()...) 36 | 37 | printTokenTree(allTokens, 0) 38 | assert.Equal(t, "{\"name\":\"abcd\",\"age\":30,\"address\":{\"city\":\"New York\",\"zip\":10001}}", TokensToString(allTokens), "JSON String is not as expected") 39 | } 40 | 41 | func TestNewJSONParserAgainstJSONBodyMissingClose(t *testing.T) { 42 | parser := NewJSONParser() 43 | testStr := "Some of the test string\n```json\n{\"name\": \"abcd\",\n\"age\": 30\n```" 44 | allTokens := make([]*Token, 0) 45 | 46 | for _, char := range testStr { 47 | allTokens = append(allTokens, parser.Parse(string(char))...) 48 | } 49 | 50 | allTokens = append(allTokens, parser.End()...) 51 | 52 | printTokenTree(allTokens, 0) 53 | assert.Equal(t, "{\"name\":\"abcd\",\"age\":30}", TokensToString(allTokens), "JSON String is not as expected") 54 | } 55 | 56 | func TestNewJSONParserAgainstJSONBodyMixedQuotes(t *testing.T) { 57 | parser := NewJSONParser() 58 | testStr := "Some of the test string\n```json\n{\"name\": \"abcd\",\n'age': 30}" 59 | allTokens := make([]*Token, 0) 60 | 61 | for _, char := range testStr { 62 | allTokens = append(allTokens, parser.Parse(string(char))...) 63 | } 64 | 65 | allTokens = append(allTokens, parser.End()...) 66 | 67 | printTokenTree(allTokens, 0) 68 | assert.Equal(t, "{\"name\":\"abcd\",\"age\":30}", TokensToString(allTokens), "JSON String is not as expected") 69 | } 70 | 71 | func TestNewJSONParserAgainstJSONBodyArrayOfPrimitives(t *testing.T) { 72 | parser := NewJSONParser() 73 | testStr := "Some of the test string\n```json\n[1, \"abcd\", true, null]" 74 | allTokens := make([]*Token, 0) 75 | 76 | for _, char := range testStr { 77 | allTokens = append(allTokens, parser.Parse(string(char))...) 78 | } 79 | 80 | allTokens = append(allTokens, parser.End()...) 81 | 82 | printTokenTree(allTokens, 0) 83 | assert.Equal(t, "[1,\"abcd\",true,null]", TokensToString(allTokens), "JSON String is not as expected") 84 | } 85 | 86 | func TestNewJSONParserAgainstJSONBodyArrayOfObjects(t *testing.T) { 87 | parser := NewJSONParser() 88 | testStr := "Some of the test string\n```json\n[{\"name\": \"abcd\",\n\"age\": 30\n}, {\"name\": \"efgh\",\n\"age\": 40\n}]" 89 | allTokens := make([]*Token, 0) 90 | 91 | for _, char := range testStr { 92 | allTokens = append(allTokens, parser.Parse(string(char))...) 93 | } 94 | 95 | allTokens = append(allTokens, parser.End()...) 96 | 97 | printTokenTree(allTokens, 0) 98 | assert.Equal(t, "[{\"name\":\"abcd\",\"age\":30},{\"name\":\"efgh\",\"age\":40}]", TokensToString(allTokens), "JSON String is not as expected") 99 | } 100 | 101 | func TestNewJSONParserAgainstJSONBodyArraySwappingNested(t *testing.T) { 102 | parser := NewJSONParser() 103 | testStr := "Some of the test string\n```json\n[{\n\"elements\": [\n{ \"elements\": [\n{\"name\": \"abcd\"}]\n}\n]\n}]" 104 | allTokens := make([]*Token, 0) 105 | 106 | for _, char := range testStr { 107 | allTokens = append(allTokens, parser.Parse(string(char))...) 108 | } 109 | 110 | allTokens = append(allTokens, parser.End()...) 111 | 112 | printTokenTree(allTokens, 0) 113 | assert.Equal(t, "[{\"elements\":[{\"elements\":[{\"name\":\"abcd\"}]}]}]", TokensToString(allTokens), "JSON String is not as expected") 114 | } 115 | 116 | func TestNewJSONParserAgainstJSONBodyArrayMissingClose(t *testing.T) { 117 | parser := NewJSONParser() 118 | testStr := "Some of the test string\n```json\n[1000,{\"name\": \"abcd\",\n\"age\": 30" 119 | allTokens := make([]*Token, 0) 120 | 121 | for _, char := range testStr { 122 | allTokens = append(allTokens, parser.Parse(string(char))...) 123 | } 124 | 125 | allTokens = append(allTokens, parser.End()...) 126 | 127 | printTokenTree(allTokens, 0) 128 | assert.Equal(t, "[1000,{\"name\":\"abcd\",\"age\":30}]", TokensToString(allTokens), "JSON String is not as expected") 129 | } 130 | 131 | func printTokenTree(tokens []*Token, indent int) { 132 | for _, token := range tokens { 133 | indentStr := strings.Repeat(" ", indent) 134 | fmt.Printf("%s%s: %s\n", indentStr, tokenTypeToString(token.Type), token.Content) //nolint:forbidigo 135 | if len(token.Children) > 0 { 136 | printTokenTree(token.Children, indent+1) 137 | } 138 | } 139 | } 140 | 141 | func tokenTypeToString(tokenType TokenType) string { 142 | switch tokenType { 143 | case TokenTypeText: 144 | return "Text" 145 | case TokenTypeJSONObject: 146 | return "Object" 147 | case TokenTypeJSONArray: 148 | return "Array" 149 | case TokenTypeJSONField: 150 | return "Field" 151 | case TokenTypeJSONString: 152 | return "String" 153 | case TokenTypeJSONNumber: 154 | return "Number" 155 | case TokenTypeJSONBoolean: 156 | return "Boolean" 157 | case TokenTypeJSONNull: 158 | return "Null" 159 | default: 160 | return "Unknown" 161 | } 162 | } 163 | -------------------------------------------------------------------------------- /pkg/semanticcache/rueidis/rueidis.go: -------------------------------------------------------------------------------- 1 | package rueidis 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "strconv" 8 | "time" 9 | 10 | "github.com/redis/rueidis" 11 | "github.com/redis/rueidis/om" 12 | "github.com/samber/lo" 13 | ) 14 | 15 | const ( 16 | retrieveTop3 int = 3 17 | retrieveTop10 int = 10 18 | ) 19 | 20 | type rueidisJSONOptions struct { 21 | dimension int 22 | } 23 | 24 | func WithDimension(dimension int) RueidisJSONCallOption { 25 | return func(o *rueidisJSONOptions) { 26 | o.dimension = dimension 27 | } 28 | } 29 | 30 | type RueidisJSONCallOption func(*rueidisJSONOptions) 31 | 32 | func applyRueidisJSONCallOptions(defaultOpts *rueidisJSONOptions, opts []RueidisJSONCallOption) *rueidisJSONOptions { 33 | for _, o := range opts { 34 | o(defaultOpts) 35 | } 36 | 37 | return defaultOpts 38 | } 39 | 40 | type Cached[T any] struct { 41 | Key string `json:"key" redis:",key"` // the redis:",key" is required to indicate which field is the ULID key 42 | Ver int64 `json:"ver" redis:",ver"` // the redis:",ver" is required to do optimistic locking to prevent lost update 43 | Vec []float64 `json:"vec"` 44 | Object T `json:"object"` 45 | } 46 | 47 | type Retrieved[T any] struct { 48 | Key string `json:"key"` 49 | Score float64 `json:"score"` 50 | Object T `json:"object"` 51 | } 52 | 53 | type SemanticCacheRueidisJSON[T any] struct { 54 | name string 55 | rueidis rueidis.Client 56 | repo om.Repository[Cached[*T]] 57 | options *rueidisJSONOptions 58 | } 59 | 60 | func RueidisJSON[T any](name string, rueidis rueidis.Client, callOptions ...RueidisJSONCallOption) *SemanticCacheRueidisJSON[T] { 61 | var t Cached[*T] 62 | 63 | opts := applyRueidisJSONCallOptions(&rueidisJSONOptions{}, callOptions) 64 | 65 | return &SemanticCacheRueidisJSON[T]{ 66 | name: name, 67 | rueidis: rueidis, 68 | repo: om.NewJSONRepository(name, t, rueidis), 69 | options: opts, 70 | } 71 | } 72 | 73 | func (c *SemanticCacheRueidisJSON[T]) newCached(doc *T, vectors []float64) *Cached[*T] { 74 | entity := c.repo.NewEntity() 75 | 76 | return &Cached[*T]{ 77 | Key: entity.Key, 78 | Ver: entity.Ver, 79 | Vec: vectors, 80 | Object: doc, 81 | } 82 | } 83 | 84 | func (c *SemanticCacheRueidisJSON[T]) CacheVectors(ctx context.Context, doc *T, vectors []float64, seconds time.Duration) (*Cached[*T], error) { 85 | cached := c.newCached(doc, vectors) 86 | 87 | err := c.repo.Save(ctx, cached) 88 | if err != nil { 89 | return nil, err 90 | } 91 | 92 | secondsNum := int64(seconds.Seconds()) 93 | if seconds >= 1 { 94 | cmd := c.rueidis.B().Expire().Key(fmt.Sprintf("%s:%s", c.name, cached.Key)).Seconds(secondsNum).Build() 95 | 96 | err = c.rueidis.Do(ctx, cmd).Error() 97 | if err != nil { 98 | return nil, err 99 | } 100 | } 101 | 102 | return cached, nil 103 | } 104 | 105 | func (c *SemanticCacheRueidisJSON[T]) createIndex(ctx context.Context, dimension int) error { 106 | dim := dimension 107 | if c.options.dimension > 0 { 108 | dim = c.options.dimension 109 | } 110 | 111 | err := c.repo.CreateIndex(ctx, func(schema om.FtCreateSchema) rueidis.Completed { 112 | return schema. 113 | FieldName("$.vec").As("vec").Vector("FLAT", 6, "TYPE", "FLOAT64", "DIM", strconv.FormatInt(int64(dim), 10), "DISTANCE_METRIC", "COSINE"). //nolint:mnd 114 | Build() 115 | }) 116 | if err != nil { 117 | return err 118 | } 119 | 120 | return nil 121 | } 122 | 123 | func (c *SemanticCacheRueidisJSON[T]) ensureIndex(ctx context.Context, dimension int) error { 124 | cmd := c.rueidis.B().FtList().Build() 125 | 126 | indexes, err := c.rueidis.Do(ctx, cmd).AsStrSlice() 127 | if err != nil { 128 | return err 129 | } 130 | 131 | indexName := c.repo.IndexName() 132 | for _, idx := range indexes { 133 | if idx == indexName { 134 | return nil 135 | } 136 | } 137 | 138 | return c.createIndex(ctx, dimension) 139 | } 140 | 141 | func (c *SemanticCacheRueidisJSON[T]) RetrieveTop3ByVectors(ctx context.Context, vectors []float64) ([]*Retrieved[*T], error) { 142 | return c.RetrieveByVectors(ctx, vectors, retrieveTop3) 143 | } 144 | 145 | func (c *SemanticCacheRueidisJSON[T]) RetrieveTop10ByVectors(ctx context.Context, vectors []float64) ([]*Retrieved[*T], error) { 146 | return c.RetrieveByVectors(ctx, vectors, retrieveTop10) 147 | } 148 | 149 | func (c *SemanticCacheRueidisJSON[T]) RetrieveFirstByVectors(ctx context.Context, vectors []float64) (*Retrieved[*T], error) { 150 | retrieved, err := c.RetrieveByVectors(ctx, vectors, 1) 151 | if err != nil { 152 | return nil, err 153 | } 154 | if len(retrieved) == 0 { 155 | return nil, nil 156 | } 157 | 158 | return retrieved[0], nil 159 | } 160 | 161 | func (c *SemanticCacheRueidisJSON[T]) RetrieveByVectors(ctx context.Context, vectors []float64, first int) ([]*Retrieved[*T], error) { 162 | err := c.ensureIndex(ctx, len(vectors)) 163 | if err != nil { 164 | return nil, err 165 | } 166 | 167 | cmd := c.rueidis.B(). 168 | FtSearch().Index(c.repo.IndexName()).Query("(*)=>[KNN "+strconv.FormatInt(int64(first), 10)+" @vec $V AS cache_score]"). 169 | Return("3").Identifier("$.object").Identifier("__vec_score").Identifier("cache_score"). 170 | Sortby("cache_score"). 171 | Params().Nargs(2).NameValue().NameValue("V", rueidis.VectorString64(vectors)). //nolint:mnd 172 | Dialect(2). //nolint:mnd 173 | Build() 174 | 175 | resp := c.rueidis.Do(ctx, cmd) 176 | 177 | _, records, err := resp.AsFtSearch() 178 | if err != nil { 179 | return nil, err 180 | } 181 | 182 | records = lo.Map(records, func(record rueidis.FtSearchDoc, _ int) rueidis.FtSearchDoc { 183 | record.Score, _ = strconv.ParseFloat(record.Doc["cache_score"], 64) 184 | return record 185 | }) 186 | 187 | return lo.Map(records, func(record rueidis.FtSearchDoc, _ int) *Retrieved[*T] { 188 | var object T 189 | _ = json.Unmarshal([]byte(record.Doc["$.object"]), &object) 190 | 191 | return &Retrieved[*T]{Key: record.Key, Score: record.Score, Object: &object} 192 | }), nil 193 | } 194 | -------------------------------------------------------------------------------- /pkg/semanticcache/rueidis/rueidis_test.go: -------------------------------------------------------------------------------- 1 | package rueidis 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "testing" 7 | "time" 8 | 9 | "github.com/lingticio/llmg/internal/datastore" 10 | "github.com/nekomeowww/xo" 11 | "github.com/samber/lo" 12 | "github.com/sashabaranov/go-openai" 13 | "github.com/stretchr/testify/assert" 14 | "github.com/stretchr/testify/require" 15 | ) 16 | 17 | type Example struct { 18 | Query string `json:"query"` 19 | Completion string `json:"completion"` 20 | } 21 | 22 | func TestSemanticCache(t *testing.T) { 23 | r, err := datastore.NewRueidis()() 24 | require.NoError(t, err) 25 | require.NotNil(t, r) 26 | 27 | json := RueidisJSON[Example]("chat_semantic_cache", r, WithDimension(1536)) 28 | 29 | config := openai.DefaultConfig(os.Getenv("OPENAI_API_KEY")) 30 | config.BaseURL = os.Getenv("OPENAI_API_BASEURL") 31 | 32 | openAI := openai.NewClientWithConfig(config) 33 | 34 | // Cache 35 | { 36 | embedding, err := openAI.CreateEmbeddings(context.Background(), openai.EmbeddingRequestStrings{ 37 | Input: []string{"When was ChatGPT released?", "Where is the headquarters of OpenAI?"}, 38 | Model: openai.AdaEmbeddingV2, 39 | EncodingFormat: openai.EmbeddingEncodingFormatFloat, 40 | }) 41 | require.NoError(t, err) 42 | require.Len(t, embedding.Data, 2) 43 | 44 | ebd1 := lo.Map(embedding.Data[0].Embedding, func(item float32, _ int) float64 { 45 | return float64(item) 46 | }) 47 | 48 | ebd2 := lo.Map(embedding.Data[1].Embedding, func(item float32, _ int) float64 { 49 | return float64(item) 50 | }) 51 | 52 | _, err = json.CacheVectors(context.Background(), &Example{Query: "Birthday of ChatGPT"}, ebd1, time.Second*2) 53 | require.NoError(t, err) 54 | 55 | _, err = json.CacheVectors(context.Background(), &Example{Query: "Apple's headquarters"}, ebd2, time.Second*2) 56 | require.NoError(t, err) 57 | } 58 | 59 | // Retrieve 60 | { 61 | embeddingQuery, err := openAI.CreateEmbeddings(context.Background(), openai.EmbeddingRequestStrings{ 62 | Input: []string{"When is the birthday of ChatGPT?"}, 63 | Model: openai.AdaEmbeddingV2, 64 | EncodingFormat: openai.EmbeddingEncodingFormatFloat, 65 | }) 66 | require.NoError(t, err) 67 | require.Len(t, embeddingQuery.Data, 1) 68 | 69 | ebdQuery := lo.Map(embeddingQuery.Data[0].Embedding, func(item float32, _ int) float64 { 70 | return float64(item) 71 | }) 72 | 73 | retrieved, err := json.RetrieveTop10ByVectors(context.Background(), ebdQuery) 74 | require.NoError(t, err) 75 | require.NotNil(t, retrieved) 76 | require.Len(t, retrieved, 2) 77 | require.NotZero(t, retrieved[0].Score) 78 | require.NotZero(t, retrieved[1].Score) 79 | 80 | assert.Equal(t, "Birthday of ChatGPT", retrieved[0].Object.Query) 81 | assert.Equal(t, "Apple's headquarters", retrieved[1].Object.Query) 82 | 83 | xo.PrintJSON(retrieved) 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /pkg/types/jwtclaim/jwtclaim.go: -------------------------------------------------------------------------------- 1 | package jwtclaim 2 | 3 | import "github.com/golang-jwt/jwt" 4 | 5 | var _ jwt.Claims = (*SupabaseAccessTokenClaim)(nil) 6 | 7 | type SupabaseAccessTokenClaim struct { 8 | Aal string `json:"aal"` 9 | Amr []SupabaseAccessTokenClaimArm `json:"amr"` 10 | AppMetadata SupabaseAccessTokenClaimAppMetadata `json:"app_metadata"` 11 | Aud any `json:"aud"` // 2024.7.12, string changed to []string 12 | Email string `json:"email"` 13 | Exp int `json:"exp"` 14 | Iat int `json:"iat"` 15 | IsAnonymous bool `json:"is_anonymous"` 16 | Iss string `json:"iss"` 17 | Phone string `json:"phone"` 18 | Role string `json:"role"` 19 | SessionID string `json:"session_id"` 20 | Sub string `json:"sub"` 21 | UserMetadata SupabaseAccessTokenClaimUserMetadata `json:"user_metadata"` 22 | } 23 | 24 | type SupabaseAccessTokenClaimArm struct { 25 | Method string `json:"method"` 26 | Timestamp int `json:"timestamp"` 27 | } 28 | 29 | type SupabaseAccessTokenClaimAppMetadata struct { 30 | Provider string `json:"provider"` 31 | Providers []string `json:"providers"` 32 | } 33 | 34 | type SupabaseAccessTokenClaimUserMetadata struct { 35 | AvatarURL string `json:"avatar_url"` 36 | Email string `json:"email"` 37 | EmailVerified bool `json:"email_verified"` 38 | FullName string `json:"full_name"` 39 | Iss string `json:"iss"` 40 | Name string `json:"name"` 41 | PhoneVerified bool `json:"phone_verified"` 42 | Picture string `json:"picture"` 43 | ProviderID string `json:"provider_id"` 44 | Sub string `json:"sub"` 45 | } 46 | 47 | func (c *SupabaseAccessTokenClaim) Valid() error { 48 | return nil 49 | } 50 | 51 | type UserProfileClaimUserMetadata struct { 52 | Type string `json:"type"` 53 | Scope string `json:"scope"` 54 | UserID string `json:"user_id"` 55 | } 56 | 57 | type UserProfileClaim struct { 58 | *jwt.StandardClaims 59 | UserMetadata UserProfileClaimUserMetadata `json:"user_metadata"` 60 | } 61 | 62 | func (c *UserProfileClaim) Valid() error { 63 | if c.StandardClaims == nil { 64 | return jwt.NewValidationError("missing standard claims", jwt.ValidationErrorClaimsInvalid) 65 | } 66 | 67 | err := c.StandardClaims.Valid() 68 | if err != nil { 69 | return err 70 | } 71 | 72 | if c.UserMetadata.Type == "" || c.UserMetadata.UserID == "" || c.UserMetadata.Scope == "" { 73 | return jwt.NewValidationError("missing user metadata", jwt.ValidationErrorClaimsInvalid) 74 | } 75 | 76 | return nil 77 | } 78 | -------------------------------------------------------------------------------- /pkg/types/metadata/metadata.go: -------------------------------------------------------------------------------- 1 | package metadata 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/samber/lo" 7 | ) 8 | 9 | var _ Identifiable = (*Tenant)(nil) 10 | var _ Identifiable = (*Team)(nil) 11 | var _ Identifiable = (*Group)(nil) 12 | 13 | type Identifiable interface { 14 | ID() string 15 | } 16 | 17 | type Tenant struct { 18 | Id string `json:"id" yaml:"id"` //nolint:stylecheck 19 | } 20 | 21 | func (t Tenant) ID() string { 22 | return t.Id 23 | } 24 | 25 | type Team struct { 26 | Id string `json:"id" yaml:"id"` //nolint:stylecheck 27 | } 28 | 29 | func (t Team) ID() string { 30 | return t.Id 31 | } 32 | 33 | type Group struct { 34 | Id string `json:"id" yaml:"id"` //nolint:stylecheck 35 | } 36 | 37 | func (g Group) ID() string { 38 | return g.Id 39 | } 40 | 41 | var _ Metadata = (*UnimplementedMetadata)(nil) 42 | 43 | type Metadata interface { 44 | Tenant() Tenant 45 | Team() Team 46 | Group() Group 47 | } 48 | 49 | type UnimplementedMetadata struct { 50 | } 51 | 52 | func (m UnimplementedMetadata) Tenant() Tenant { 53 | return Tenant{} 54 | } 55 | 56 | func (m UnimplementedMetadata) Team() Team { 57 | return Team{} 58 | } 59 | 60 | func (m UnimplementedMetadata) Group() Group { 61 | return Group{} 62 | } 63 | 64 | type UpstreamOpenAICompatibleChat struct { 65 | Usage bool `json:"usage" yaml:"usage"` 66 | Stream bool `json:"stream" yaml:"stream"` 67 | } 68 | 69 | type UpstreamOpenAICompatible struct { 70 | Chat UpstreamOpenAICompatibleChat `json:"chat" yaml:"chat"` 71 | Models bool `json:"models" yaml:"models"` 72 | Embeddings bool `json:"embeddings" yaml:"embeddings"` 73 | Images bool `json:"images" yaml:"images"` 74 | Audio bool `json:"audio" yaml:"audio"` 75 | } 76 | 77 | type UpstreamOpenAI struct { 78 | Weight *uint `json:"weight" yaml:"weight"` 79 | 80 | BaseURL string `json:"base_url" yaml:"base_url"` 81 | APIKey string `json:"api_key" yaml:"api_key"` 82 | ExtraHeaders http.Header `json:"extra_headers" yaml:"extra_headers"` 83 | Compatible UpstreamOpenAICompatible `json:"compatible" yaml:"compatible"` 84 | } 85 | 86 | var _ Upstreamable = (*Upstream)(nil) 87 | var _ Upstreamable = (*Upstreams)(nil) 88 | var _ Upstreamable = (*UpstreamSingleOrMultiple)(nil) 89 | 90 | type Upstreamable interface { 91 | IsSingleUpstream() bool 92 | GetUpstream() *Upstream 93 | GetUpstreams() []*Upstream 94 | } 95 | 96 | type Upstream struct { 97 | OpenAI UpstreamOpenAI `json:"openai" yaml:"openai"` 98 | } 99 | 100 | func (*Upstream) IsSingleUpstream() bool { 101 | return true 102 | } 103 | 104 | func (u *Upstream) GetUpstream() *Upstream { 105 | return u 106 | } 107 | 108 | func (u *Upstream) GetUpstreams() []*Upstream { 109 | return []*Upstream{u} 110 | } 111 | 112 | type Upstreams []*Upstream 113 | 114 | func (Upstreams) IsSingleUpstream() bool { 115 | return false 116 | } 117 | 118 | func (u Upstreams) GetUpstream() *Upstream { 119 | if len(u) == 0 { 120 | return nil 121 | } 122 | 123 | return u[0] 124 | } 125 | 126 | func (u Upstreams) GetUpstreams() []*Upstream { 127 | return lo.Map(u, func(item *Upstream, index int) *Upstream { 128 | return item 129 | }) 130 | } 131 | 132 | type UpstreamSingleOrMultiple struct { 133 | *Upstream `yaml:",inline"` 134 | 135 | Group Upstreams `json:"group" yaml:"group"` 136 | } 137 | 138 | func (u *UpstreamSingleOrMultiple) IsSingleUpstream() bool { 139 | return len(u.Group) == 0 140 | } 141 | 142 | func (u *UpstreamSingleOrMultiple) GetUpstream() *Upstream { 143 | if u.IsSingleUpstream() { 144 | return u.Upstream 145 | } 146 | 147 | return u.Group.GetUpstream() 148 | } 149 | 150 | func (u *UpstreamSingleOrMultiple) GetUpstreams() []*Upstream { 151 | if u.IsSingleUpstream() { 152 | return []*Upstream{u.Upstream} 153 | } 154 | 155 | return u.Group.GetUpstreams() 156 | } 157 | -------------------------------------------------------------------------------- /pkg/types/redis/rediskeys/keys.go: -------------------------------------------------------------------------------- 1 | package rediskeys 2 | 3 | import "fmt" 4 | 5 | // Key key. 6 | type Key string 7 | 8 | // Format format. 9 | func (k Key) Format(params ...interface{}) string { 10 | return fmt.Sprintf(string(k), params...) 11 | } 12 | 13 | // Endpoint Provider 14 | 15 | const ( 16 | // EndpointMetadataByAPIKey1. 17 | // Params: API Key. 18 | EndpointMetadataByAPIKey1 Key = "config:providers:auth:metadata:api_key:%s" 19 | 20 | // EndpointUpstreamByTenantID1. 21 | // Params: Tenant ID. 22 | EndpointUpstreamByTenantID1 Key = "config:providers:auth:metadata:upstream:tenant:%s" 23 | 24 | // EndpointUpstreamByTeamID1. 25 | // Params: Team ID. 26 | EndpointUpstreamByTeamID1 Key = "config:providers:auth:metadata:upstream:team:%s" 27 | 28 | // EndpointUpstreamByGroupID1. 29 | // Params: Group ID. 30 | EndpointUpstreamByGroupID1 Key = "config:providers:auth:metadata:upstream:group:%s" 31 | 32 | // EndpointUpstreamByEndpointID1. 33 | // Params: Endpoint ID. 34 | EndpointUpstreamByEndpointID1 Key = "config:providers:auth:metadata:upstream:endpoint:%s" 35 | 36 | // EndpointMetadataByAlias1. 37 | // Params: Alias. 38 | EndpointMetadataByAlias1 Key = "config:providers:auth:metadata:alias:%s" 39 | ) 40 | -------------------------------------------------------------------------------- /pkg/util/eventsource/eventsource.go: -------------------------------------------------------------------------------- 1 | package eventsource 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | 10 | "github.com/labstack/echo/v4" 11 | "github.com/lingticio/llmg/pkg/util/nanoid" 12 | ) 13 | 14 | // Event represents Server-Sent Event. 15 | // SSE explanation: https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#event_stream_format 16 | type Event struct { 17 | // ID is used to set the EventSource object's last event ID value. 18 | ID []byte 19 | // Data field is for the message. When the EventSource receives multiple consecutive lines 20 | // that begin with data:, it concatenates them, inserting a newline character between each one. 21 | // Trailing newlines are removed. 22 | Data []byte 23 | // Event is a string identifying the type of event described. If this is specified, an event 24 | // will be dispatched on the browser to the listener for the specified event name; the website 25 | // source code should use addEventListener() to listen for named events. The onmessage handler 26 | // is called if no event name is specified for a message. 27 | Event []byte 28 | // Retry is the reconnection time. If the connection to the server is lost, the browser will 29 | // wait for the specified time before attempting to reconnect. This must be an integer, specifying 30 | // the reconnection time in milliseconds. If a non-integer value is specified, the field is ignored. 31 | Retry []byte 32 | // Comment line can be used to prevent connections from timing out; a server can send a comment 33 | // periodically to keep the connection alive. 34 | Comment []byte 35 | } 36 | 37 | // MarshalTo marshals Event to given Writer. 38 | func (ev *Event) MarshalTo(w io.Writer) error { 39 | // Marshalling part is taken from: https://github.com/r3labs/sse/blob/c6d5381ee3ca63828b321c16baa008fd6c0b4564/http.go#L16 40 | if len(ev.Data) == 0 && len(ev.Comment) == 0 { 41 | return nil 42 | } 43 | 44 | if len(ev.Data) > 0 { //nolint 45 | if len(ev.ID) > 0 { 46 | if _, err := fmt.Fprintf(w, "id: %s\n", ev.ID); err != nil { 47 | return err 48 | } 49 | } 50 | 51 | sd := bytes.Split(ev.Data, []byte("\n")) 52 | for i := range sd { 53 | if _, err := fmt.Fprintf(w, "data: %s\n", sd[i]); err != nil { 54 | return err 55 | } 56 | } 57 | 58 | if len(ev.Event) > 0 { 59 | if _, err := fmt.Fprintf(w, "event: %s\n", ev.Event); err != nil { 60 | return err 61 | } 62 | } 63 | 64 | if len(ev.Retry) > 0 { 65 | if _, err := fmt.Fprintf(w, "retry: %s\n", ev.Retry); err != nil { 66 | return err 67 | } 68 | } 69 | } 70 | 71 | if len(ev.Comment) > 0 { 72 | if _, err := fmt.Fprintf(w, ": %s\n", ev.Comment); err != nil { 73 | return err 74 | } 75 | } 76 | 77 | if _, err := fmt.Fprint(w, "\n"); err != nil { 78 | return err 79 | } 80 | 81 | return nil 82 | } 83 | 84 | type ResponseAdapterType string 85 | 86 | const ( 87 | ResponseAdapterTypeEcho ResponseAdapterType = "echo" 88 | ) 89 | 90 | type options struct { 91 | id string 92 | idGenerateHandler func() string 93 | 94 | responseType ResponseAdapterType 95 | echoResponse *echo.Response 96 | } 97 | 98 | type CallOption func(*options) 99 | 100 | func WithEchoResponse(response *echo.Response) CallOption { 101 | return func(o *options) { 102 | o.responseType = ResponseAdapterTypeEcho 103 | o.echoResponse = response 104 | } 105 | } 106 | 107 | func WithID(id string) CallOption { 108 | return func(o *options) { 109 | o.id = id 110 | } 111 | } 112 | 113 | func WithIDGenerator(handler func() string) CallOption { 114 | return func(o *options) { 115 | o.idGenerateHandler = handler 116 | } 117 | } 118 | 119 | type EventSource[D any] struct { 120 | options *options 121 | } 122 | 123 | func NewEventSource[D any](callOpts ...CallOption) *EventSource[D] { 124 | opts := &options{ 125 | idGenerateHandler: func() string { 126 | return nanoid.New() 127 | }, 128 | } 129 | 130 | for _, opt := range callOpts { 131 | opt(opts) 132 | } 133 | 134 | if opts.id == "" { 135 | opts.id = opts.idGenerateHandler() 136 | } 137 | 138 | es := &EventSource[D]{ 139 | options: opts, 140 | } 141 | 142 | switch opts.responseType { 143 | case ResponseAdapterTypeEcho: 144 | ApplyHeaders(opts.echoResponse.Header()) 145 | } 146 | 147 | return es 148 | } 149 | 150 | func ApplyHeaders(header http.Header) { 151 | header.Set("Content-Type", "text/event-stream") 152 | header.Set("Cache-Control", "no-cache") 153 | header.Set("Connection", "keep-alive") 154 | } 155 | 156 | func (e *EventSource[D]) SendJSON(message D) error { 157 | jsonData, err := json.Marshal(message) 158 | if err != nil { 159 | return err 160 | } 161 | 162 | event := Event{ 163 | Data: jsonData, 164 | } 165 | 166 | switch e.options.responseType { 167 | case ResponseAdapterTypeEcho: 168 | err = event.MarshalTo(e.options.echoResponse.Writer) 169 | if err != nil { 170 | return err 171 | } 172 | 173 | e.options.echoResponse.Flush() 174 | } 175 | 176 | return nil 177 | } 178 | 179 | func (e *EventSource[D]) SendRaw(message []byte) error { 180 | event := Event{ 181 | Data: message, 182 | } 183 | 184 | switch e.options.responseType { 185 | case ResponseAdapterTypeEcho: 186 | err := event.MarshalTo(e.options.echoResponse.Writer) 187 | if err != nil { 188 | return err 189 | } 190 | 191 | e.options.echoResponse.Flush() 192 | } 193 | 194 | return nil 195 | } 196 | 197 | func (e *EventSource[D]) SendWithID(id string, message D) error { 198 | jsonData, err := json.Marshal(message) 199 | if err != nil { 200 | return err 201 | } 202 | 203 | event := Event{ 204 | ID: []byte(id), 205 | Data: jsonData, 206 | } 207 | 208 | switch e.options.responseType { 209 | case ResponseAdapterTypeEcho: 210 | err = event.MarshalTo(e.options.echoResponse.Writer) 211 | if err != nil { 212 | return err 213 | } 214 | 215 | e.options.echoResponse.Flush() 216 | } 217 | 218 | return nil 219 | } 220 | -------------------------------------------------------------------------------- /pkg/util/grpc/gateway.go: -------------------------------------------------------------------------------- 1 | package grpc 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | 7 | "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" 8 | "github.com/nekomeowww/xo/logger" 9 | "google.golang.org/grpc" 10 | ) 11 | 12 | type gatewayOptions struct { 13 | serverMuxOptions []runtime.ServeMuxOption 14 | handlers []func(ctx context.Context, serveMux *runtime.ServeMux, clientConn *grpc.ClientConn) error 15 | } 16 | 17 | type GatewayCallOption func(*gatewayOptions) 18 | 19 | func WithServerMuxOptions(opts ...runtime.ServeMuxOption) GatewayCallOption { 20 | return func(o *gatewayOptions) { 21 | o.serverMuxOptions = append(o.serverMuxOptions, opts...) 22 | } 23 | } 24 | 25 | func WithHandlers(handlers ...func(ctx context.Context, serveMux *runtime.ServeMux, clientConn *grpc.ClientConn) error) GatewayCallOption { 26 | return func(o *gatewayOptions) { 27 | o.handlers = append(o.handlers, handlers...) 28 | } 29 | } 30 | 31 | func NewGateway( 32 | ctx context.Context, 33 | conn *grpc.ClientConn, 34 | logger *logger.Logger, 35 | callOpts ...GatewayCallOption, 36 | ) (http.Handler, error) { 37 | opts := &gatewayOptions{} 38 | 39 | for _, f := range callOpts { 40 | f(opts) 41 | } 42 | 43 | mux := runtime.NewServeMux(opts.serverMuxOptions...) 44 | 45 | for _, f := range opts.handlers { 46 | if err := f(ctx, mux, conn); err != nil { 47 | return nil, err 48 | } 49 | } 50 | 51 | return mux, nil 52 | } 53 | -------------------------------------------------------------------------------- /pkg/util/grpc/register.go: -------------------------------------------------------------------------------- 1 | package grpc 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" 7 | "github.com/labstack/echo/v4" 8 | "google.golang.org/grpc" 9 | "google.golang.org/grpc/reflection" 10 | ) 11 | 12 | type HTTPHandler = func(ctx context.Context, serveMux *runtime.ServeMux, clientConn *grpc.ClientConn) error 13 | type GRPCServiceRegister func(s reflection.GRPCServer) 14 | 15 | type Register struct { 16 | HTTPHandlers []HTTPHandler 17 | GrpcServices []GRPCServiceRegister 18 | EchoHandlers map[string]map[string]echo.HandlerFunc 19 | } 20 | 21 | func NewRegister() *Register { 22 | return &Register{ 23 | HTTPHandlers: make([]HTTPHandler, 0), 24 | GrpcServices: make([]GRPCServiceRegister, 0), 25 | EchoHandlers: make(map[string]map[string]echo.HandlerFunc), 26 | } 27 | } 28 | 29 | func (r *Register) RegisterHTTPHandler(handler HTTPHandler) { 30 | r.HTTPHandlers = append(r.HTTPHandlers, handler) 31 | } 32 | 33 | func (r *Register) RegisterHTTPHandlers(handlers []HTTPHandler) { 34 | r.HTTPHandlers = append(r.HTTPHandlers, handlers...) 35 | } 36 | 37 | func (r *Register) RegisterGrpcService(serviceRegister GRPCServiceRegister) { 38 | r.GrpcServices = append(r.GrpcServices, serviceRegister) 39 | } 40 | 41 | func (r *Register) RegisterEchoHandler(path string, method string, handler echo.HandlerFunc) { 42 | if _, ok := r.EchoHandlers[path]; !ok { 43 | r.EchoHandlers[path] = make(map[string]echo.HandlerFunc) 44 | } 45 | 46 | r.EchoHandlers[path][method] = handler 47 | } 48 | -------------------------------------------------------------------------------- /pkg/util/nanoid/nanoid.go: -------------------------------------------------------------------------------- 1 | package nanoid 2 | 3 | import ( 4 | gonanoid "github.com/matoous/go-nanoid/v2" 5 | "github.com/samber/lo" 6 | ) 7 | 8 | func New() string { 9 | return lo.Must(gonanoid.Generate("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz", 8)) //nolint:mnd 10 | } 11 | 12 | func NewWithLength(length int) string { 13 | return lo.Must(gonanoid.Generate("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz", length)) 14 | } 15 | -------------------------------------------------------------------------------- /pkg/util/utils/file.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | func ImageExtensionFromContentType(contentType string) string { 4 | switch contentType { 5 | case "image/jpeg": 6 | return "jpg" 7 | case "image/png": 8 | return "png" 9 | case "image/gif": 10 | return "gif" 11 | default: 12 | return "jpg" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /pkg/util/utils/path.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import "net/url" 4 | 5 | func Path(str string) string { 6 | parsed, err := url.Parse(str) 7 | if err != nil { 8 | return "" 9 | } 10 | 11 | return parsed.Path 12 | } 13 | 14 | func Host(str string) string { 15 | parsed, err := url.Parse(str) 16 | if err != nil { 17 | return "" 18 | } 19 | 20 | return parsed.Host 21 | } 22 | 23 | func Schema(str string) string { 24 | parsed, err := url.Parse(str) 25 | if err != nil { 26 | return "" 27 | } 28 | 29 | return parsed.Scheme 30 | } 31 | -------------------------------------------------------------------------------- /pkg/util/utils/string.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "unicode" 5 | 6 | "github.com/rivo/uniseg" 7 | ) 8 | 9 | func IsStringAcceptable(str string) bool { 10 | for _, v := range str { 11 | if v == '\n' || v == '\r' || v == '\t' { 12 | continue 13 | } 14 | if !unicode.IsGraphic(v) { 15 | return false 16 | } 17 | } 18 | 19 | return true 20 | } 21 | 22 | func UnPrintable(s string) []rune { 23 | var unPrintable []rune 24 | 25 | for _, r := range s { 26 | if r == '\n' || r == '\r' || r == '\t' { 27 | continue 28 | } 29 | if !unicode.IsPrint(r) { 30 | unPrintable = append(unPrintable, r) 31 | } 32 | } 33 | 34 | return unPrintable 35 | } 36 | 37 | // CharacterCount counts one ASCII character is counted as one character, other characters are counted as two (including emoji). 38 | func CharacterCount(str string) int { 39 | var count int 40 | 41 | grIterator := uniseg.NewGraphemes(str) 42 | for grIterator.Next() { 43 | r := grIterator.Runes() 44 | if len(r) > 1 || r[0] > unicode.MaxASCII { 45 | count += 2 46 | } else { 47 | count++ 48 | } 49 | } 50 | 51 | return count 52 | } 53 | -------------------------------------------------------------------------------- /pkg/util/utils/unittest.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "os" 5 | "strings" 6 | ) 7 | 8 | func IsInUnitTest() bool { 9 | for _, arg := range os.Args { 10 | if strings.HasPrefix(arg, "-test.") { 11 | return true 12 | } 13 | } 14 | 15 | return false 16 | } 17 | -------------------------------------------------------------------------------- /tools/tools.go: -------------------------------------------------------------------------------- 1 | //go:build tools 2 | 3 | package tools 4 | 5 | import ( 6 | _ "github.com/maxbrunsfeld/counterfeiter/v6" 7 | _ "github.com/99designs/gqlgen" 8 | _ "github.com/99designs/gqlgen/graphql/introspection" 9 | ) 10 | --------------------------------------------------------------------------------