├── .dockerignore ├── images ├── logs.png ├── runs.png ├── syncs.png ├── clear_state.png ├── connectors.png ├── endpoints.png ├── syncs_edit.png ├── syncs_create.png ├── connectors_edit.png └── endpoints_create.png ├── cosmos-frontend ├── jsconfig.json ├── public │ ├── favicon.ico │ └── index.html ├── babel.config.js ├── src │ ├── assets │ │ ├── logo.png │ │ ├── material-design-gray-and-blue.jpg │ │ └── logo.svg │ ├── styles │ │ └── variables.scss │ ├── store │ │ └── index.js │ ├── App.vue │ ├── supabase │ │ └── index.js │ ├── main.js │ ├── views │ │ ├── Endpoints.vue │ │ ├── Connectors.vue │ │ ├── Home.vue │ │ ├── Artifacts.vue │ │ ├── ConnectorsByType.vue │ │ ├── EndpointsByType.vue │ │ ├── Syncs.vue │ │ └── Runs.vue │ ├── axios │ │ └── index.js │ ├── plugins │ │ └── vuetify.js │ ├── router │ │ └── index.js │ └── components │ │ ├── ConfirmationDialog.vue │ │ ├── Navbar.vue │ │ ├── CreateConnector.vue │ │ ├── EditConnector.vue │ │ └── CreateEndpoint.vue ├── .env.production ├── .env.development ├── vue.config.js └── package.json ├── .env ├── cosmos-backend ├── scheduler.go ├── worker.go ├── postgres │ ├── db.go │ ├── connector.go │ ├── run.go │ ├── endpoint.go │ ├── sync.go │ ├── postgres.go │ └── migrations │ │ └── 0000000000.sql ├── log.go ├── go.mod ├── cosmos.go ├── command.go ├── http │ ├── artifact.go │ ├── run.go │ ├── connector.go │ ├── sync.go │ ├── endpoint.go │ └── server.go ├── artifact.go ├── error.go ├── cmd │ ├── temporald │ │ └── main.go │ └── cosmosd │ │ └── main.go ├── zap │ └── log.go ├── temporal │ └── worker.go ├── scheduler │ └── scheduler.go ├── filesystem │ └── artifact.go ├── run.go ├── connector.go ├── endpoint.go ├── message.go ├── sync.go └── form.go ├── Makefile ├── .gitignore ├── vetur.config.js ├── Dockerfile ├── LICENSE ├── docker-compose.yaml └── README.md /.dockerignore: -------------------------------------------------------------------------------- 1 | **/node_modules 2 | -------------------------------------------------------------------------------- /images/logs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/varunbpatil/cosmos/HEAD/images/logs.png -------------------------------------------------------------------------------- /images/runs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/varunbpatil/cosmos/HEAD/images/runs.png -------------------------------------------------------------------------------- /images/syncs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/varunbpatil/cosmos/HEAD/images/syncs.png -------------------------------------------------------------------------------- /cosmos-frontend/jsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "include": [ 3 | "./src/**/*" 4 | ] 5 | } 6 | -------------------------------------------------------------------------------- /images/clear_state.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/varunbpatil/cosmos/HEAD/images/clear_state.png -------------------------------------------------------------------------------- /images/connectors.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/varunbpatil/cosmos/HEAD/images/connectors.png -------------------------------------------------------------------------------- /images/endpoints.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/varunbpatil/cosmos/HEAD/images/endpoints.png -------------------------------------------------------------------------------- /images/syncs_edit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/varunbpatil/cosmos/HEAD/images/syncs_edit.png -------------------------------------------------------------------------------- /images/syncs_create.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/varunbpatil/cosmos/HEAD/images/syncs_create.png -------------------------------------------------------------------------------- /images/connectors_edit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/varunbpatil/cosmos/HEAD/images/connectors_edit.png -------------------------------------------------------------------------------- /images/endpoints_create.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/varunbpatil/cosmos/HEAD/images/endpoints_create.png -------------------------------------------------------------------------------- /.env: -------------------------------------------------------------------------------- 1 | ARTIFACT_DIR=/tmp/cosmos/artifacts 2 | SCRATCH_SPACE=/tmp/cosmos/scratch 3 | LOCAL_DIR=/tmp/cosmos/local 4 | -------------------------------------------------------------------------------- /cosmos-frontend/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/varunbpatil/cosmos/HEAD/cosmos-frontend/public/favicon.ico -------------------------------------------------------------------------------- /cosmos-frontend/babel.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | presets: [ 3 | '@vue/cli-plugin-babel/preset' 4 | ] 5 | } 6 | -------------------------------------------------------------------------------- /cosmos-frontend/src/assets/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/varunbpatil/cosmos/HEAD/cosmos-frontend/src/assets/logo.png -------------------------------------------------------------------------------- /cosmos-frontend/.env.production: -------------------------------------------------------------------------------- 1 | VUE_APP_API_ROOT = 'http://localhost:5000' 2 | VUE_APP_SUPABASE_REALTIME_URL = 'ws://localhost:4000/socket' 3 | -------------------------------------------------------------------------------- /cosmos-frontend/.env.development: -------------------------------------------------------------------------------- 1 | VUE_APP_API_ROOT = 'http://localhost:5000' 2 | VUE_APP_SUPABASE_REALTIME_URL = 'ws://localhost:4000/socket' 3 | -------------------------------------------------------------------------------- /cosmos-backend/scheduler.go: -------------------------------------------------------------------------------- 1 | package cosmos 2 | 3 | type SchedulerService interface { 4 | Schedule(syncID *int, runOptions *RunOptions) error 5 | } 6 | -------------------------------------------------------------------------------- /cosmos-backend/worker.go: -------------------------------------------------------------------------------- 1 | package cosmos 2 | 3 | import "context" 4 | 5 | type WorkerService interface { 6 | CancelRun(ctx context.Context, runID int) error 7 | } 8 | -------------------------------------------------------------------------------- /cosmos-frontend/src/assets/material-design-gray-and-blue.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/varunbpatil/cosmos/HEAD/cosmos-frontend/src/assets/material-design-gray-and-blue.jpg -------------------------------------------------------------------------------- /cosmos-frontend/vue.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | transpileDependencies: [ 3 | 'vuetify' 4 | ], 5 | configureWebpack: { 6 | performance: { 7 | hints: false 8 | } 9 | }, 10 | } 11 | -------------------------------------------------------------------------------- /cosmos-frontend/src/styles/variables.scss: -------------------------------------------------------------------------------- 1 | // Globals 2 | $body-font-family: 'Roboto', sans-serif; 3 | 4 | $card-border-radius: 0; 5 | $dialog-border-radius: 0; 6 | $menu-content-border-radius: 0; 7 | $menu-content-elevation: 4; 8 | -------------------------------------------------------------------------------- /cosmos-frontend/src/store/index.js: -------------------------------------------------------------------------------- 1 | import Vue from 'vue' 2 | import Vuex from 'vuex' 3 | 4 | Vue.use(Vuex) 5 | 6 | export default new Vuex.Store({ 7 | state: { 8 | }, 9 | mutations: { 10 | }, 11 | actions: { 12 | }, 13 | modules: { 14 | } 15 | }) 16 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | build: 2 | @echo "Building cosmos-frontend..." 3 | cd cosmos-frontend && npm run build 4 | @echo "Building docker image..." 5 | docker build --no-cache -t varunpatil/cosmos:0.1.7 . 6 | 7 | clean: 8 | rm -rf cosmos-frontend/dist 9 | 10 | .PHONY: build clean 11 | -------------------------------------------------------------------------------- /cosmos-backend/postgres/db.go: -------------------------------------------------------------------------------- 1 | package postgres 2 | 3 | import "cosmos" 4 | 5 | var _ cosmos.DBService = (*DBService)(nil) 6 | 7 | type DBService struct { 8 | db *DB 9 | } 10 | 11 | func NewDBService(db *DB) *DBService { 12 | return &DBService{ 13 | db: db, 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /cosmos-backend/log.go: -------------------------------------------------------------------------------- 1 | package cosmos 2 | 3 | type Logger interface { 4 | Debug(msg string, keyvals ...interface{}) 5 | Info(msg string, keyvals ...interface{}) 6 | Warn(msg string, keyvals ...interface{}) 7 | Error(msg string, keyvals ...interface{}) 8 | WithKV(keyvals ...interface{}) Logger 9 | } 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | node_modules 3 | dist 4 | 5 | # local env files 6 | .env.local 7 | .env.*.local 8 | 9 | # Log files 10 | npm-debug.log* 11 | yarn-debug.log* 12 | yarn-error.log* 13 | pnpm-debug.log* 14 | 15 | # Editor directories and files 16 | .idea 17 | .vscode 18 | *.suo 19 | *.ntvs* 20 | *.njsproj 21 | *.sln 22 | *.sw? 23 | -------------------------------------------------------------------------------- /vetur.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('vls').VeturConfig} */ 2 | module.exports = { 3 | // override vscode settings 4 | // Notice: It only affects the settings used by Vetur. 5 | settings: { 6 | "vetur.useWorkspaceDependencies": true, 7 | "vetur.experimental.templateInterpolationService": false 8 | }, 9 | // support monorepos 10 | projects: [ 11 | './cosmos-frontend' 12 | ] 13 | } 14 | -------------------------------------------------------------------------------- /cosmos-backend/go.mod: -------------------------------------------------------------------------------- 1 | module cosmos 2 | 3 | go 1.16 4 | 5 | require ( 6 | github.com/gorilla/handlers v1.5.1 7 | github.com/gorilla/mux v1.8.0 8 | github.com/iancoleman/orderedmap v0.2.0 9 | github.com/jackc/pgx/v4 v4.11.0 10 | github.com/json-iterator/go v1.1.11 11 | github.com/mitchellh/mapstructure v1.4.1 12 | github.com/xeipuuv/gojsonschema v1.2.0 13 | go.temporal.io/sdk v1.8.0 14 | go.uber.org/zap v1.13.0 15 | ) 16 | -------------------------------------------------------------------------------- /cosmos-frontend/src/App.vue: -------------------------------------------------------------------------------- 1 | 11 | 12 | 27 | 28 | 33 | -------------------------------------------------------------------------------- /cosmos-frontend/src/assets/logo.svg: -------------------------------------------------------------------------------- 1 | Artboard 46 2 | -------------------------------------------------------------------------------- /cosmos-backend/cosmos.go: -------------------------------------------------------------------------------- 1 | package cosmos 2 | 3 | import ( 4 | jsoniter "github.com/json-iterator/go" 5 | ) 6 | 7 | var json = jsoniter.ConfigDefault 8 | 9 | const ( 10 | ScratchSpace = "/tmp/cosmos/scratch" 11 | TemporalTaskQueue = "cosmos-task-queue" 12 | ) 13 | 14 | type DBService interface { 15 | ConnectorService 16 | EndpointService 17 | SyncService 18 | RunService 19 | } 20 | 21 | type App struct { 22 | DBService 23 | CommandService 24 | MessageService 25 | ArtifactService 26 | SchedulerService 27 | WorkerService 28 | Logger 29 | } 30 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:alpine AS builder 2 | WORKDIR /cosmos-backend 3 | COPY ./cosmos-backend/ . 4 | RUN CGO_ENABLED=0 go build ./cmd/cosmosd/ 5 | RUN CGO_ENABLED=0 go build ./cmd/temporald/ 6 | 7 | FROM alpine:3.14 8 | ADD https://github.com/ufoscout/docker-compose-wait/releases/download/2.9.0/wait /wait 9 | RUN chmod +x /wait 10 | RUN apk update && apk add --no-cache docker-cli tini 11 | COPY --from=builder /cosmos-backend/cosmosd /cosmosd 12 | COPY --from=builder /cosmos-backend/temporald /temporald 13 | COPY ./cosmos-frontend/dist /dist 14 | ENTRYPOINT ["/sbin/tini", "-g", "--"] -------------------------------------------------------------------------------- /cosmos-backend/command.go: -------------------------------------------------------------------------------- 1 | package cosmos 2 | 3 | import ( 4 | "context" 5 | ) 6 | 7 | type CommandService interface { 8 | Spec(ctx context.Context, connector *Connector) (*Message, error) 9 | Check(ctx context.Context, connector *Connector, config interface{}) (*Message, error) 10 | Discover(ctx context.Context, connector *Connector, config interface{}) (*Message, error) 11 | Read(ctx context.Context, connector *Connector, empty bool) (<-chan interface{}, <-chan error) 12 | Write(ctx context.Context, connector *Connector, in <-chan *Message) (<-chan interface{}, <-chan error) 13 | Normalize(ctx context.Context, connector *Connector, basicNormalization bool) (<-chan interface{}, <-chan error) 14 | } 15 | -------------------------------------------------------------------------------- /cosmos-frontend/src/supabase/index.js: -------------------------------------------------------------------------------- 1 | import { RealtimeClient } from '@supabase/realtime-js' 2 | 3 | const client = new RealtimeClient(process.env.VUE_APP_SUPABASE_REALTIME_URL) 4 | client.connect() 5 | 6 | const ConnectorChanges = client.channel(`realtime:public:connectors`) 7 | const EndpointChanges = client.channel(`realtime:public:endpoints`) 8 | const SyncChanges = client.channel(`realtime:public:syncs`) 9 | const RunChanges = client.channel(`realtime:public:runs`) 10 | 11 | ConnectorChanges.subscribe() 12 | EndpointChanges.subscribe() 13 | SyncChanges.subscribe() 14 | RunChanges.subscribe() 15 | 16 | export { 17 | ConnectorChanges, 18 | EndpointChanges, 19 | SyncChanges, 20 | RunChanges 21 | } -------------------------------------------------------------------------------- /cosmos-frontend/src/main.js: -------------------------------------------------------------------------------- 1 | import Vue from 'vue' 2 | import App from './App.vue' 3 | import store from './store' 4 | import vuetify from './plugins/vuetify' 5 | import router from './router' 6 | import axios from './axios' 7 | import { 8 | ConnectorChanges, 9 | EndpointChanges, 10 | SyncChanges, 11 | RunChanges 12 | } from './supabase' 13 | 14 | Vue.config.productionTip = false 15 | 16 | Vue.prototype.$axios = axios 17 | Vue.prototype.$connectorChanges = ConnectorChanges 18 | Vue.prototype.$endpointChanges = EndpointChanges 19 | Vue.prototype.$syncChanges = SyncChanges 20 | Vue.prototype.$runChanges = RunChanges 21 | 22 | new Vue({ 23 | store, 24 | vuetify, 25 | router, 26 | render: h => h(App) 27 | }).$mount('#app') 28 | -------------------------------------------------------------------------------- /cosmos-frontend/src/views/Endpoints.vue: -------------------------------------------------------------------------------- 1 | 19 | 20 | 25 | -------------------------------------------------------------------------------- /cosmos-frontend/src/views/Connectors.vue: -------------------------------------------------------------------------------- 1 | 19 | 20 | 25 | -------------------------------------------------------------------------------- /cosmos-frontend/src/axios/index.js: -------------------------------------------------------------------------------- 1 | import axios from 'axios' 2 | 3 | // Create an axios instance with a custom baseURL. 4 | // This is useful during development when the backend API server is running on a different port. 5 | // VUE_APP_API_ROOT is set in the .env.development file inside the project root. 6 | // The environment variable must have the VUE_APP_* prefix. 7 | // See https://stackoverflow.com/questions/47407564/change-the-default-base-url-for-axios 8 | // https://cli.vuejs.org/guide/mode-and-env.html#modes 9 | // https://forum.vuejs.org/t/accessing-axios-in-vuex-module/29414/3 10 | // https://github.com/axios/axios#custom-instance-defaults 11 | const VueAxios = axios.create() 12 | VueAxios.defaults.baseURL = process.env.VUE_APP_API_ROOT 13 | 14 | export default VueAxios 15 | -------------------------------------------------------------------------------- /cosmos-frontend/src/views/Home.vue: -------------------------------------------------------------------------------- 1 | 12 | 13 | 18 | 19 | 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Airbyte 4 | Copyright (c) 2021 Varun B Patil 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | -------------------------------------------------------------------------------- /cosmos-backend/http/artifact.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "cosmos" 5 | "fmt" 6 | "net/http" 7 | "strconv" 8 | 9 | "github.com/gorilla/mux" 10 | ) 11 | 12 | func (s *Server) registerArtifactRoutes(r *mux.Router) { 13 | r.HandleFunc("/artifacts/{runID}/{artifactID}", s.getArtifact).Methods("GET") 14 | } 15 | 16 | func (s *Server) getArtifact(w http.ResponseWriter, r *http.Request) { 17 | runID, err := strconv.Atoi(mux.Vars(r)["runID"]) 18 | if err != nil { 19 | s.ReplyWithSanitizedError(w, r, cosmos.Errorf(cosmos.EINVALID, "Invalid run ID")) 20 | return 21 | } 22 | artifactID, err := strconv.Atoi(mux.Vars(r)["artifactID"]) 23 | if err != nil { 24 | s.ReplyWithSanitizedError(w, r, cosmos.Errorf(cosmos.EINVALID, "Invalid artifact ID")) 25 | return 26 | } 27 | 28 | run, err := s.App.FindRunByID(r.Context(), runID) 29 | if err != nil { 30 | s.ReplyWithSanitizedError(w, r, err) 31 | return 32 | } 33 | 34 | artifactory, err := s.App.GetArtifactory(run.SyncID, run.ExecutionDate) 35 | if err != nil { 36 | s.ReplyWithSanitizedError(w, r, err) 37 | return 38 | } 39 | 40 | data, err := s.App.GetArtifactData(artifactory, artifactID) 41 | if err != nil { 42 | s.ReplyWithSanitizedError(w, r, err) 43 | return 44 | } 45 | 46 | fmt.Fprint(w, string(data)) 47 | } 48 | -------------------------------------------------------------------------------- /cosmos-frontend/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | Cosmos 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 19 |
20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /cosmos-frontend/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cosmos-ui", 3 | "version": "0.1.0", 4 | "private": true, 5 | "scripts": { 6 | "serve": "vue-cli-service serve", 7 | "build": "vue-cli-service build", 8 | "lint": "vue-cli-service lint" 9 | }, 10 | "dependencies": { 11 | "@supabase/realtime-js": "^1.1.1", 12 | "ansispan": "0.0.4", 13 | "axios": "^0.21.1", 14 | "colors": "^1.4.0", 15 | "core-js": "^3.6.5", 16 | "date-fns": "^2.21.1", 17 | "lodash": "^4.17.21", 18 | "vue": "^2.6.11", 19 | "vue-router": "^3.2.0", 20 | "vuetify": "^2.5.6", 21 | "vuex": "^3.4.0" 22 | }, 23 | "devDependencies": { 24 | "@vue/cli-plugin-babel": "~4.5.0", 25 | "@vue/cli-plugin-eslint": "~4.5.0", 26 | "@vue/cli-plugin-router": "^4.5.11", 27 | "@vue/cli-plugin-vuex": "^4.5.11", 28 | "@vue/cli-service": "~4.5.0", 29 | "babel-eslint": "^10.1.0", 30 | "eslint": "^6.8.0", 31 | "eslint-plugin-vue": "^6.2.2", 32 | "sass": "^1.32.13", 33 | "sass-loader": "^10.0.0", 34 | "vue-cli-plugin-vuetify": "~2.2.2", 35 | "vue-template-compiler": "^2.6.11", 36 | "vuetify-loader": "^1.7.0" 37 | }, 38 | "eslintConfig": { 39 | "root": true, 40 | "env": { 41 | "node": true 42 | }, 43 | "extends": [ 44 | "plugin:vue/essential", 45 | "eslint:recommended" 46 | ], 47 | "parserOptions": { 48 | "parser": "babel-eslint" 49 | }, 50 | "rules": {} 51 | }, 52 | "browserslist": [ 53 | "> 1%", 54 | "last 2 versions", 55 | "not dead" 56 | ] 57 | } 58 | -------------------------------------------------------------------------------- /cosmos-frontend/src/plugins/vuetify.js: -------------------------------------------------------------------------------- 1 | import Vue from 'vue'; 2 | import Vuetify from 'vuetify/lib/framework'; 3 | import { 4 | VApp, 5 | VAutocomplete, 6 | VAvatar, 7 | VBtn, 8 | VCard, 9 | VCheckbox, 10 | VCol, 11 | VContainer, 12 | VDatePicker, 13 | VDialog, 14 | VDivider, 15 | VIcon, 16 | VList, 17 | VMain, 18 | VMenu, 19 | VNavigationDrawer, 20 | VPagination, 21 | VProgressLinear, 22 | VRow, 23 | VSelect, 24 | VSnackbar, 25 | VSpacer, 26 | VSwitch, 27 | VTab, 28 | VTabs, 29 | VTextField, 30 | VToolbar, 31 | VTooltip, 32 | } from 'vuetify/lib'; 33 | import { 34 | Ripple 35 | } from 'vuetify/lib/directives'; 36 | 37 | Vue.use(Vuetify, { 38 | // Any new vuetify component used needs to be added here. 39 | // Otherwise, you'll get "min-css-extract-plugin" conflicting order errors. 40 | // See https://stackoverflow.com/a/64419994 41 | components: { 42 | VApp, 43 | VAutocomplete, 44 | VAvatar, 45 | VBtn, 46 | VCard, 47 | VCheckbox, 48 | VCol, 49 | VContainer, 50 | VDatePicker, 51 | VDialog, 52 | VDivider, 53 | VIcon, 54 | VList, 55 | VMain, 56 | VMenu, 57 | VNavigationDrawer, 58 | VPagination, 59 | VProgressLinear, 60 | VRow, 61 | VSelect, 62 | VSnackbar, 63 | VSpacer, 64 | VSwitch, 65 | VTab, 66 | VTabs, 67 | VTextField, 68 | VToolbar, 69 | VTooltip, 70 | }, 71 | directives: { 72 | Ripple, 73 | }, 74 | }) 75 | 76 | export default new Vuetify({ 77 | }); 78 | -------------------------------------------------------------------------------- /cosmos-backend/artifact.go: -------------------------------------------------------------------------------- 1 | package cosmos 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "sync" 7 | "time" 8 | ) 9 | 10 | type ctxKey string 11 | 12 | const ( 13 | ArtifactDir = "/tmp/cosmos/artifacts" 14 | artifactoryKey ctxKey = "artifactory" 15 | ) 16 | 17 | const ( 18 | ArtifactSource = iota 19 | ArtifactDestination 20 | ArtifactNormalization 21 | ArtifactWorker 22 | ArtifactSrcConfig 23 | ArtifactDstConfig 24 | ArtifactSrcCatalog 25 | ArtifactDstCatalog 26 | ArtifactBeforeState 27 | ArtifactAfterState 28 | ArtifactMax 29 | ) 30 | 31 | var ArtifactNames = [ArtifactMax]string{ 32 | "source", 33 | "destination", 34 | "normalization", 35 | "worker", 36 | "source-config", 37 | "destination-config", 38 | "source-catalog", 39 | "destination-catalog", 40 | "before-state", 41 | "after-state", 42 | } 43 | 44 | type Artifactory struct { 45 | Path string 46 | Once [ArtifactMax]sync.Once 47 | Artifacts [ArtifactMax]*log.Logger 48 | } 49 | 50 | type ArtifactService interface { 51 | GetArtifactory(syncID int, executionDate time.Time) (*Artifactory, error) 52 | GetArtifactRef(artifactory *Artifactory, id int, attempt int32) (*log.Logger, error) 53 | WriteArtifact(artifactory *Artifactory, id int, contents interface{}) error 54 | GetArtifactPath(artifactory *Artifactory, id int) *string 55 | GetArtifactData(artifactory *Artifactory, id int) ([]byte, error) 56 | CloseArtifactory(artifactory *Artifactory) 57 | } 58 | 59 | func NewArtifactoryContext(ctx context.Context, artifactory *Artifactory) context.Context { 60 | return context.WithValue(ctx, artifactoryKey, artifactory) 61 | } 62 | 63 | func ArtifactoryFromContext(ctx context.Context) *Artifactory { 64 | return ctx.Value(artifactoryKey).(*Artifactory) 65 | } 66 | -------------------------------------------------------------------------------- /cosmos-backend/error.go: -------------------------------------------------------------------------------- 1 | package cosmos 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | ) 7 | 8 | // Application error codes. 9 | const ( 10 | EINVALID = "invalid" 11 | EINTERNAL = "internal" 12 | ECONFLICT = "conflict" 13 | ENOTFOUND = "not_found" 14 | ENOTIMPLEMENTED = "not_implemented" 15 | ) 16 | 17 | // Error represents an application-specific error. 18 | type Error struct { 19 | Code string 20 | Message string 21 | } 22 | 23 | func (e *Error) Error() string { 24 | return fmt.Sprintf("cosmos error: code=%s message=%s", e.Code, e.Message) 25 | } 26 | 27 | // ErrorCode unwraps an application error and returns its code. 28 | // Non-application errors always return EINTERNAL to avoid leaking 29 | // sensitive implementation details to the end user. 30 | func ErrorCode(err error) string { 31 | var e *Error 32 | if errors.As(err, &e) { 33 | return e.Code 34 | } 35 | return EINTERNAL 36 | } 37 | 38 | // ErrorMessage unwraps an application error and returns its message. 39 | // Non-application errors always return "Internal error" to avoid leaking 40 | // sensitive implementation details to the end user. 41 | func ErrorMessage(err error) string { 42 | var e *Error 43 | if errors.As(err, &e) { 44 | return e.Message 45 | } 46 | return "Internal error" 47 | } 48 | 49 | // Errorf is a helper function to create an application-specific error. 50 | func Errorf(code string, format string, args ...interface{}) *Error { 51 | // Do not create internal errors. Any error that is not application-specific 52 | // is, by default, an internal error. 53 | if code == EINTERNAL { 54 | panic("internal errors are not application-specific errors") 55 | } 56 | 57 | return &Error{ 58 | Code: code, 59 | Message: fmt.Sprintf(format, args...), 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /cosmos-backend/cmd/temporald/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "cosmos" 5 | "cosmos/docker" 6 | "cosmos/filesystem" 7 | "cosmos/jsonschema" 8 | "cosmos/postgres" 9 | "cosmos/temporal" 10 | "cosmos/zap" 11 | "log" 12 | 13 | "go.temporal.io/sdk/client" 14 | "go.temporal.io/sdk/worker" 15 | ) 16 | 17 | func main() { 18 | client, err := client.NewClient(client.Options{HostPort: "temporal:7233", Logger: zap.NewLogger()}) 19 | if err != nil { 20 | log.Fatal("Unable to create temporal client. err: " + err.Error()) 21 | } 22 | defer client.Close() 23 | 24 | db := postgres.NewDB("postgres://postgres:password@postgresql:5432/postgres", false) 25 | dbService := postgres.NewDBService(db) 26 | messageService := jsonschema.NewMessageService() 27 | artifactService := filesystem.NewArtifactService() 28 | commandService := docker.NewCommandService() 29 | workflow := temporal.NewWorkflow() 30 | app := &cosmos.App{ 31 | DBService: dbService, 32 | CommandService: commandService, 33 | MessageService: messageService, 34 | ArtifactService: artifactService, 35 | } 36 | commandService.App = app 37 | workflow.App = app 38 | 39 | if err := db.Open(); err != nil { 40 | log.Fatal("Unable to connect to db in temporal worker. err: " + err.Error()) 41 | } 42 | defer db.Close() 43 | 44 | w := worker.New(client, cosmos.TemporalTaskQueue, worker.Options{}) 45 | w.RegisterWorkflow(workflow.IngestionWorkflow) 46 | w.RegisterActivity(workflow.GetRun) 47 | w.RegisterActivity(workflow.Initialize) 48 | w.RegisterActivity(workflow.ReplicationActivity) 49 | w.RegisterActivity(workflow.NormalizationActivity) 50 | w.RegisterActivity(workflow.DBUpdateActivity) 51 | 52 | if err := w.Run(worker.InterruptCh()); err != nil { 53 | log.Fatal("Unable to start temporal worker. err: " + err.Error()) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /cosmos-backend/http/run.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "cosmos" 5 | "net/http" 6 | "strconv" 7 | 8 | "github.com/gorilla/mux" 9 | ) 10 | 11 | func (s *Server) registerRunRoutes(r *mux.Router) { 12 | r.HandleFunc("/runs", s.findRuns).Methods("POST") 13 | r.HandleFunc("/runs/{id}", s.findRuns).Methods("GET") 14 | 15 | r.HandleFunc("/runs/{id}/cancel", s.cancelRun).Methods("POST") 16 | } 17 | 18 | func (s *Server) findRuns(w http.ResponseWriter, r *http.Request) { 19 | filter := cosmos.RunFilter{} 20 | 21 | switch r.Method { 22 | case http.MethodPost: 23 | if err := json.NewDecoder(r.Body).Decode(&filter); err != nil { 24 | s.ReplyWithSanitizedError(w, r, cosmos.Errorf(cosmos.EINVALID, "Invalid JSON body")) 25 | return 26 | } 27 | case http.MethodGet: 28 | runID, err := strconv.Atoi(mux.Vars(r)["id"]) 29 | if err != nil { 30 | s.ReplyWithSanitizedError(w, r, cosmos.Errorf(cosmos.EINVALID, "Invalid run ID")) 31 | return 32 | } 33 | filter.ID = &runID 34 | default: 35 | panic("Unhandled request method in findRuns") 36 | } 37 | 38 | runs, totalRuns, err := s.App.FindRuns(r.Context(), filter) 39 | if err != nil { 40 | s.ReplyWithSanitizedError(w, r, err) 41 | return 42 | } 43 | 44 | ret := map[string]interface{}{ 45 | "runs": runs, 46 | "totalRuns": totalRuns, 47 | } 48 | 49 | w.Header().Set("Content-Type", "application/json") 50 | if err := json.NewEncoder(w).Encode(&ret); err != nil { 51 | s.LogError(r, err) 52 | } 53 | } 54 | 55 | func (s *Server) cancelRun(w http.ResponseWriter, r *http.Request) { 56 | runID, err := strconv.Atoi(mux.Vars(r)["id"]) 57 | if err != nil { 58 | s.ReplyWithSanitizedError(w, r, cosmos.Errorf(cosmos.EINVALID, "Invalid run ID")) 59 | return 60 | } 61 | if err := s.App.CancelRun(r.Context(), runID); err != nil { 62 | s.ReplyWithSanitizedError(w, r, err) 63 | return 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /cosmos-frontend/src/router/index.js: -------------------------------------------------------------------------------- 1 | import Vue from 'vue' 2 | import VueRouter from 'vue-router' 3 | import Home from '../views/Home.vue' 4 | 5 | Vue.use(VueRouter) 6 | 7 | const routes = [ 8 | { 9 | path: '/', 10 | name: 'Home', 11 | component: Home 12 | }, 13 | { 14 | path: '/connectors', 15 | redirect: '/connectors/sources', // redirect to the "sources" child route by default. 16 | name: 'Connectors', 17 | // route level code-splitting 18 | // this generates a separate chunk (about.[hash].js) for this route 19 | // which is lazy-loaded when the route is visited. 20 | component: () => import(/* webpackChunkName: "connectors" */ '../views/Connectors.vue'), 21 | children: [ 22 | { 23 | path: ':type', 24 | component: () => import(/* webpackChunkName: "connectorsbytype" */ '../views/ConnectorsByType.vue'), 25 | } 26 | ] 27 | }, 28 | { 29 | path: '/endpoints', 30 | redirect: '/endpoints/sources', 31 | name: 'Endpoints', 32 | component: () => import(/* webpackChunkName: "endpoints" */ '../views/Endpoints.vue'), 33 | children: [ 34 | { 35 | path: ':type', 36 | component: () => import(/* webpackChunkName: "connectorsbytype" */ '../views/EndpointsByType.vue'), 37 | } 38 | ] 39 | }, 40 | { 41 | path: '/syncs', 42 | name: 'Syncs', 43 | component: () => import(/* webpackChunkName: "syncs" */ '../views/Syncs.vue'), 44 | children: [ 45 | { 46 | path: ':syncID', 47 | name: 'Runs', 48 | component: () => import(/* webpackChunkName: "runs" */ '../views/Runs.vue'), 49 | children: [ 50 | { 51 | path: ':runID', 52 | name: 'Artifacts', 53 | component: () => import(/* webpackChunkName: "artifacts" */ '../views/Artifacts.vue'), 54 | } 55 | ] 56 | } 57 | ] 58 | } 59 | ] 60 | 61 | const router = new VueRouter({ 62 | mode: 'history', 63 | base: process.env.BASE_URL, 64 | routes 65 | }) 66 | 67 | export default router 68 | -------------------------------------------------------------------------------- /cosmos-backend/zap/log.go: -------------------------------------------------------------------------------- 1 | package zap 2 | 3 | import ( 4 | "cosmos" 5 | "encoding/json" 6 | 7 | "go.temporal.io/sdk/log" 8 | "go.uber.org/zap" 9 | ) 10 | 11 | var ( 12 | _ cosmos.Logger = (*Logger)(nil) 13 | _ log.Logger = (*Logger)(nil) 14 | _ log.WithLogger = (*Logger)(nil) 15 | ) 16 | 17 | type Logger struct { 18 | *zap.SugaredLogger 19 | } 20 | 21 | func NewLogger() *Logger { 22 | rawJSON := []byte(`{ 23 | "level": "debug", 24 | "encoding": "json", 25 | "outputPaths": ["stdout", "/tmp/logs"], 26 | "errorOutputPaths": ["stderr"], 27 | "encoderConfig": { 28 | "messageKey": "message", 29 | "levelKey": "level", 30 | "levelEncoder": "lowercase", 31 | "timeKey": "timestamp", 32 | "timeEncoder": "rfc3339" 33 | } 34 | }`) 35 | 36 | var cfg zap.Config 37 | if err := json.Unmarshal(rawJSON, &cfg); err != nil { 38 | panic("failed to unmarshal zap config. err: " + err.Error()) 39 | } 40 | 41 | logger, err := cfg.Build() 42 | if err != nil { 43 | panic("failed to create a zap logger. err: " + err.Error()) 44 | } 45 | 46 | return &Logger{logger.Sugar()} 47 | } 48 | 49 | func (l *Logger) Debug(msg string, keyvals ...interface{}) { 50 | l.SugaredLogger.Debugw(msg, keyvals...) 51 | } 52 | 53 | func (l *Logger) Info(msg string, keyvals ...interface{}) { 54 | l.SugaredLogger.Infow(msg, keyvals...) 55 | } 56 | 57 | func (l *Logger) Warn(msg string, keyvals ...interface{}) { 58 | l.SugaredLogger.Warnw(msg, keyvals...) 59 | } 60 | 61 | func (l *Logger) Error(msg string, keyvals ...interface{}) { 62 | l.SugaredLogger.Errorw(msg, keyvals...) 63 | } 64 | 65 | func (l *Logger) WithKV(keyvals ...interface{}) cosmos.Logger { 66 | return &Logger{l.SugaredLogger.With(keyvals...)} 67 | } 68 | 69 | // This method is there purely so that this logger can be used in Temporal. 70 | // Inside an activity, you can get a cosmos.Logger using activity.GetLogger(ctx).(cosmos.Logger). 71 | // However, inside a workflow, you will not be able to get a cosmos.Logger because workflows use 72 | // a ReplayLogger to be deterministic. 73 | func (l *Logger) With(keyvals ...interface{}) log.Logger { 74 | return &Logger{l.SugaredLogger.With(keyvals...)} 75 | } 76 | -------------------------------------------------------------------------------- /cosmos-frontend/src/components/ConfirmationDialog.vue: -------------------------------------------------------------------------------- 1 | 40 | 41 | 82 | -------------------------------------------------------------------------------- /cosmos-frontend/src/components/Navbar.vue: -------------------------------------------------------------------------------- 1 | 52 | 53 | 67 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: "3.7" 2 | x-golang-daemon: &golang-daemon 3 | image: varunpatil/cosmos:0.1.7 4 | depends_on: 5 | - postgresql 6 | - temporal 7 | environment: 8 | WAIT_HOSTS: postgresql:5432, temporal:7233 9 | WAIT_TIMEOUT: 120 10 | WAIT_AFTER: 10 11 | ARTIFACT_DIR: ${ARTIFACT_DIR} 12 | SCRATCH_SPACE: ${SCRATCH_SPACE} 13 | LOCAL_DIR: ${LOCAL_DIR} 14 | volumes: 15 | - /var/run/docker.sock:/var/run/docker.sock 16 | - ${ARTIFACT_DIR}:/tmp/cosmos/artifacts 17 | - ${SCRATCH_SPACE}:/tmp/cosmos/scratch 18 | - ${LOCAL_DIR}:/tmp/cosmos/local 19 | restart: unless-stopped 20 | services: 21 | postgresql: 22 | container_name: cosmos-postgresql 23 | environment: 24 | POSTGRES_PASSWORD: password 25 | image: postgres:13 26 | command: -c wal_level=logical -c max_replication_slots=1 27 | networks: 28 | - cosmos-network 29 | ports: 30 | - 5432:5432 31 | volumes: 32 | - postgres-data:/var/lib/postgresql/data 33 | restart: unless-stopped 34 | temporal: 35 | container_name: cosmos-temporal 36 | depends_on: 37 | - postgresql 38 | environment: 39 | - DB=postgresql 40 | - DB_PORT=5432 41 | - POSTGRES_USER=postgres 42 | - POSTGRES_PWD=password 43 | - POSTGRES_SEEDS=postgresql 44 | - DYNAMIC_CONFIG_FILE_PATH=config/dynamicconfig/development.yaml 45 | image: temporalio/auto-setup:1.10.5 46 | networks: 47 | - cosmos-network 48 | ports: 49 | - 7233:7233 50 | restart: unless-stopped 51 | cosmos: 52 | container_name: cosmos 53 | <<: *golang-daemon 54 | command: sh -c '/wait && /cosmosd' 55 | networks: 56 | - cosmos-network 57 | ports: 58 | - 5000:5000 59 | cosmos-worker: 60 | container_name: cosmos-temporal-worker 61 | <<: *golang-daemon 62 | command: sh -c '/wait && /temporald' 63 | networks: 64 | - cosmos-network 65 | supabase-realtime: 66 | container_name: supabase-realtime-server 67 | image: supabase/realtime 68 | environment: 69 | - DB_HOST=postgresql 70 | - DB_NAME=postgres 71 | - DB_USER=postgres 72 | - DB_PASSWORD=password 73 | - DB_PORT=5432 74 | - SECURE_CHANNELS=false 75 | - PORT=4000 76 | depends_on: 77 | - cosmos 78 | - postgresql 79 | networks: 80 | - cosmos-network 81 | ports: 82 | - 4000:4000 83 | restart: unless-stopped 84 | networks: 85 | cosmos-network: 86 | volumes: 87 | postgres-data: 88 | -------------------------------------------------------------------------------- /cosmos-backend/temporal/worker.go: -------------------------------------------------------------------------------- 1 | package temporal 2 | 3 | import ( 4 | "context" 5 | "cosmos" 6 | "log" 7 | "runtime/debug" 8 | "strconv" 9 | "sync" 10 | "time" 11 | 12 | "go.temporal.io/sdk/client" 13 | ) 14 | 15 | var _ cosmos.WorkerService = (*Worker)(nil) 16 | 17 | type Worker struct { 18 | ctx context.Context 19 | cancel context.CancelFunc 20 | wg sync.WaitGroup 21 | 22 | client.Client 23 | *cosmos.App 24 | } 25 | 26 | func NewWorker() *Worker { 27 | ctx, cancel := context.WithCancel(context.Background()) 28 | return &Worker{ 29 | ctx: ctx, 30 | cancel: cancel, 31 | } 32 | } 33 | 34 | func (w *Worker) Open() error { 35 | w.wg.Add(1) 36 | go w.WorkerLoop(w.ctx) 37 | return nil 38 | } 39 | 40 | func (w *Worker) Close() error { 41 | w.cancel() 42 | w.wg.Wait() 43 | return nil 44 | } 45 | 46 | func (w *Worker) CancelRun(ctx context.Context, runID int) error { 47 | run, err := w.App.FindRunByID(ctx, runID) 48 | if err != nil { 49 | return err 50 | } 51 | return w.Client.CancelWorkflow(ctx, run.TemporalWorkflowID, run.TemporalRunID) 52 | } 53 | 54 | func recoverFromPanic() { 55 | if err := recover(); err != nil { 56 | log.Printf("worker panic: %s", err) 57 | debug.PrintStack() 58 | } 59 | } 60 | 61 | func (w *Worker) WorkerLoop(ctx context.Context) { 62 | defer w.wg.Done() 63 | 64 | for { 65 | select { 66 | case <-ctx.Done(): 67 | return 68 | case <-time.After(3 * time.Second): 69 | w.DoWork() 70 | } 71 | } 72 | } 73 | 74 | func (w *Worker) DoWork() { 75 | defer recoverFromPanic() 76 | 77 | runs, _, err := w.App.FindRuns(w.ctx, cosmos.RunFilter{Status: []string{cosmos.RunStatusQueued}}) 78 | if err != nil { 79 | log.Printf("worker err: %s", err) 80 | return 81 | } 82 | 83 | for _, run := range runs { 84 | options := client.StartWorkflowOptions{ID: strconv.Itoa(run.SyncID), TaskQueue: cosmos.TemporalTaskQueue} 85 | 86 | // If there is already a workflow running, ExecuteWorkflow() will simply return its run id without creating a new one. 87 | wr, err := w.Client.ExecuteWorkflow(w.ctx, options, NewWorkflow().IngestionWorkflow, run.ID) 88 | if err != nil { 89 | log.Printf("worker failed to start temporal workflow. err: %s", err) 90 | continue 91 | } 92 | temporalWorkflowID := wr.GetID() 93 | temporalRunID := wr.GetRunID() 94 | 95 | // Don't set the status to "running" here. It will be set in the workflow. 96 | // Even if this UpdateRun fails, ExecuteWorkflow() will return the same run id next time around. 97 | run, err = w.App.UpdateRun( 98 | w.ctx, 99 | run.ID, 100 | &cosmos.RunUpdate{ 101 | TemporalWorkflowID: &temporalWorkflowID, 102 | TemporalRunID: &temporalRunID, 103 | }, 104 | ) 105 | if err != nil { 106 | log.Printf("worker err: %s", err) 107 | } 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /cosmos-backend/scheduler/scheduler.go: -------------------------------------------------------------------------------- 1 | package scheduler 2 | 3 | import ( 4 | "context" 5 | "cosmos" 6 | "errors" 7 | "log" 8 | "runtime/debug" 9 | "sync" 10 | "time" 11 | ) 12 | 13 | var _ cosmos.SchedulerService = (*Scheduler)(nil) 14 | 15 | type Scheduler struct { 16 | ctx context.Context 17 | cancel context.CancelFunc 18 | sync.Mutex 19 | sync.WaitGroup 20 | 21 | *cosmos.App 22 | } 23 | 24 | func NewScheduler() *Scheduler { 25 | ctx, cancel := context.WithCancel(context.Background()) 26 | return &Scheduler{ 27 | ctx: ctx, 28 | cancel: cancel, 29 | } 30 | } 31 | 32 | func (s *Scheduler) Open() error { 33 | s.Add(1) 34 | go s.SchedulerLoop(s.ctx) 35 | return nil 36 | } 37 | 38 | func (s *Scheduler) Close() error { 39 | s.cancel() 40 | s.Wait() 41 | return nil 42 | } 43 | 44 | func recoverFromPanic() { 45 | if err := recover(); err != nil { 46 | log.Printf("scheduler panic: %s", err) 47 | debug.PrintStack() 48 | } 49 | } 50 | 51 | func (s *Scheduler) SchedulerLoop(ctx context.Context) { 52 | defer s.Done() 53 | 54 | for { 55 | select { 56 | case <-ctx.Done(): 57 | return 58 | case <-time.After(3 * time.Second): 59 | s.Schedule(nil, &cosmos.RunOptions{}) 60 | } 61 | } 62 | } 63 | 64 | func (s *Scheduler) Schedule(syncID *int, runOptions *cosmos.RunOptions) error { 65 | defer recoverFromPanic() 66 | 67 | s.Lock() 68 | defer s.Unlock() 69 | 70 | ctx, cancel := context.WithCancel(s.ctx) 71 | defer cancel() 72 | 73 | syncs, _, err := s.App.FindSyncs(ctx, cosmos.SyncFilter{ID: syncID}) 74 | if err != nil { 75 | log.Printf("scheduler err: %s", err) 76 | return err 77 | } 78 | 79 | for _, sync := range syncs { 80 | run, err := s.App.GetLastRunForSyncID(ctx, sync.ID) 81 | if err != nil && !errors.Is(err, cosmos.ErrNoPrevRun) { 82 | log.Printf("scheduler err: %s", err) 83 | if syncID != nil { 84 | return err 85 | } 86 | continue 87 | } 88 | 89 | ok, err := okToSchedule(sync, run, syncID != nil) 90 | if !ok { 91 | if err != nil && syncID != nil { 92 | return err 93 | } 94 | continue 95 | } 96 | 97 | run = &cosmos.Run{SyncID: sync.ID, ExecutionDate: time.Now(), Options: *runOptions} 98 | if err := s.App.CreateRun(ctx, run); err != nil { 99 | log.Printf("scheduler err: %s", err) 100 | } 101 | } 102 | 103 | return nil 104 | } 105 | 106 | func okToSchedule(sync *cosmos.Sync, run *cosmos.Run, force bool) (bool, error) { 107 | if !sync.Enabled && !force { 108 | return false, cosmos.Errorf(cosmos.ECONFLICT, "Not enabled") 109 | } 110 | if run == nil { 111 | // No previous run. 112 | return true, nil 113 | } 114 | if time.Now().Sub(run.ExecutionDate) < time.Duration(sync.ScheduleInterval)*time.Minute && !force { 115 | return false, cosmos.Errorf(cosmos.ECONFLICT, "Interval has not elapsed") 116 | } 117 | if !run.IsTerminalState() { 118 | return false, cosmos.Errorf(cosmos.ECONFLICT, "A run is in progress") 119 | } 120 | return true, nil 121 | } 122 | -------------------------------------------------------------------------------- /cosmos-frontend/src/views/Artifacts.vue: -------------------------------------------------------------------------------- 1 | 33 | 34 | 109 | -------------------------------------------------------------------------------- /cosmos-backend/filesystem/artifact.go: -------------------------------------------------------------------------------- 1 | package filesystem 2 | 3 | import ( 4 | "cosmos" 5 | "fmt" 6 | "io/ioutil" 7 | "log" 8 | "os" 9 | "path/filepath" 10 | "reflect" 11 | "strconv" 12 | "strings" 13 | "time" 14 | 15 | jsoniter "github.com/json-iterator/go" 16 | ) 17 | 18 | var json = jsoniter.ConfigDefault 19 | 20 | var _ cosmos.ArtifactService = (*ArtifactService)(nil) 21 | 22 | type ArtifactService struct { 23 | } 24 | 25 | func NewArtifactService() *ArtifactService { 26 | return &ArtifactService{} 27 | } 28 | 29 | func (s *ArtifactService) GetArtifactory(syncID int, executionDate time.Time) (*cosmos.Artifactory, error) { 30 | path := filepath.Join( 31 | cosmos.ArtifactDir, 32 | strconv.Itoa(syncID), 33 | executionDate.Format(time.RFC3339), 34 | ) 35 | 36 | if err := os.MkdirAll(path, 0777); err != nil { 37 | return nil, err 38 | } 39 | 40 | return &cosmos.Artifactory{Path: path}, nil 41 | } 42 | 43 | func (s *ArtifactService) GetArtifactRef(artifactory *cosmos.Artifactory, id int, attempt int32) (*log.Logger, error) { 44 | var err error 45 | 46 | artifactory.Once[id].Do(func() { 47 | var file *os.File 48 | file, err = os.OpenFile(filepath.Join(artifactory.Path, cosmos.ArtifactNames[id]), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) 49 | if err != nil { 50 | return 51 | } 52 | artifactory.Artifacts[id] = log.New(file, fmt.Sprintf("[Attempt %3d] ", attempt), log.LstdFlags) 53 | }) 54 | 55 | if err != nil { 56 | return nil, err 57 | } 58 | 59 | if artifactory.Artifacts[id] == nil { 60 | return nil, fmt.Errorf("artifact for %s is unavailable", cosmos.ArtifactNames[id]) 61 | } 62 | 63 | return artifactory.Artifacts[id], nil 64 | } 65 | 66 | func (s *ArtifactService) WriteArtifact(artifactory *cosmos.Artifactory, id int, contents interface{}) error { 67 | if reflect.ValueOf(contents).IsNil() { 68 | return nil 69 | } 70 | 71 | file, err := os.Create(filepath.Join(artifactory.Path, cosmos.ArtifactNames[id])) 72 | if err != nil { 73 | return err 74 | } 75 | defer file.Close() 76 | 77 | b, err := json.Marshal(contents) 78 | if err != nil { 79 | return err 80 | } 81 | 82 | _, err = file.Write(b) 83 | if err != nil { 84 | return err 85 | } 86 | 87 | return nil 88 | } 89 | 90 | func (s *ArtifactService) GetArtifactPath(artifactory *cosmos.Artifactory, id int) *string { 91 | path := filepath.Join(artifactory.Path, cosmos.ArtifactNames[id]) 92 | if _, err := os.Stat(path); os.IsNotExist(err) { 93 | return nil 94 | } 95 | 96 | // For Docker-in-Docker, we have to return the path as it would be on the host. 97 | path = strings.TrimPrefix(path, cosmos.ArtifactDir) 98 | path = filepath.Join(os.Getenv("ARTIFACT_DIR"), path) 99 | 100 | return &path 101 | } 102 | 103 | func (s *ArtifactService) GetArtifactData(artifactory *cosmos.Artifactory, id int) ([]byte, error) { 104 | path := filepath.Join(artifactory.Path, cosmos.ArtifactNames[id]) 105 | 106 | if _, err := os.Stat(path); err != nil { 107 | if os.IsNotExist(err) { 108 | return nil, cosmos.Errorf(cosmos.ENOTFOUND, "Requested artifact does not exist") 109 | } 110 | return nil, err 111 | } 112 | 113 | return ioutil.ReadFile(path) 114 | } 115 | 116 | func (s *ArtifactService) CloseArtifactory(artifactory *cosmos.Artifactory) { 117 | for _, artifact := range artifactory.Artifacts { 118 | if artifact != nil { 119 | artifact.Writer().(*os.File).Close() 120 | } 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /cosmos-backend/run.go: -------------------------------------------------------------------------------- 1 | package cosmos 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "time" 7 | ) 8 | 9 | const ( 10 | RunStatusQueued = "queued" 11 | RunStatusRunning = "running" 12 | RunStatusSuccess = "success" 13 | RunStatusFailed = "failed" 14 | RunStatusCanceled = "canceled" 15 | RunStatusWiped = "wiped" 16 | ) 17 | 18 | var ( 19 | ErrNoPrevRun = errors.New("no previous run of this sync") 20 | ) 21 | 22 | type Run struct { 23 | ID int `json:"id"` 24 | SyncID int `json:"syncID"` 25 | ExecutionDate time.Time `json:"executionDate"` 26 | Status string `json:"status"` 27 | Stats RunStats `json:"stats"` 28 | Options RunOptions `json:"options"` 29 | TemporalWorkflowID string `json:"temporalWorkflowID"` 30 | TemporalRunID string `json:"temporalRunID"` 31 | Sync *Sync `json:"sync"` 32 | } 33 | 34 | func (r *Run) IsTerminalState() bool { 35 | switch r.Status { 36 | case RunStatusSuccess, RunStatusFailed, RunStatusCanceled, RunStatusWiped: 37 | return true 38 | default: 39 | return false 40 | } 41 | } 42 | 43 | type RunStats struct { 44 | NumRecords uint64 `json:"numRecords"` 45 | ExecutionStart time.Time `json:"executionStart"` 46 | ExecutionEnd time.Time `json:"executionEnd"` 47 | } 48 | 49 | type RunOptions struct { 50 | WipeDestination bool `json:"wipeDestination"` 51 | } 52 | 53 | type RunUpdate struct { 54 | Status *string `json:"status"` 55 | Retries *int `json:"retries"` 56 | NumRecords *uint64 `json:"numRecords"` 57 | ExecutionStart *time.Time `json:"executionStart"` 58 | ExecutionEnd *time.Time `json:"executionEnd"` 59 | Options *RunOptions `json:"options"` 60 | TemporalWorkflowID *string `json:"temporalWorkflowID"` 61 | TemporalRunID *string `json:"temporalRunID"` 62 | } 63 | 64 | type RunFilter struct { 65 | ID *int `json:"id"` 66 | SyncID *int `json:"syncID"` 67 | Status []string `json:"status"` 68 | DateRange []string `json:"dateRange"` 69 | 70 | Offset int `json:"offset"` 71 | Limit int `json:"limit"` 72 | } 73 | 74 | type RunService interface { 75 | FindRunByID(ctx context.Context, id int) (*Run, error) 76 | FindRuns(ctx context.Context, filter RunFilter) ([]*Run, int, error) 77 | CreateRun(ctx context.Context, run *Run) error 78 | UpdateRun(ctx context.Context, id int, run *Run) error 79 | GetLastRunForSyncID(ctx context.Context, syncID int) (*Run, error) 80 | } 81 | 82 | func (a *App) UpdateRun(ctx context.Context, id int, upd *RunUpdate) (*Run, error) { 83 | run, err := a.FindRunByID(ctx, id) 84 | if err != nil { 85 | return nil, err 86 | } 87 | 88 | // Update fields if set. 89 | if v := upd.Status; v != nil { 90 | run.Status = *v 91 | } 92 | if v := upd.NumRecords; v != nil { 93 | run.Stats.NumRecords = *v 94 | } 95 | if v := upd.ExecutionStart; v != nil { 96 | run.Stats.ExecutionStart = *v 97 | } 98 | if v := upd.ExecutionEnd; v != nil { 99 | run.Stats.ExecutionEnd = *v 100 | } 101 | if v := upd.Options; v != nil { 102 | run.Options = *v 103 | } 104 | if v := upd.TemporalWorkflowID; v != nil { 105 | run.TemporalWorkflowID = *v 106 | } 107 | if v := upd.TemporalRunID; v != nil { 108 | run.TemporalRunID = *v 109 | } 110 | 111 | if err := a.DBService.UpdateRun(ctx, id, run); err != nil { 112 | return nil, err 113 | } 114 | 115 | return run, nil 116 | } 117 | -------------------------------------------------------------------------------- /cosmos-backend/cmd/cosmosd/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "cosmos" 5 | "cosmos/docker" 6 | "cosmos/filesystem" 7 | "cosmos/http" 8 | "cosmos/jsonschema" 9 | "cosmos/postgres" 10 | "cosmos/scheduler" 11 | "cosmos/temporal" 12 | "cosmos/zap" 13 | "fmt" 14 | "log" 15 | "os" 16 | "os/signal" 17 | 18 | "go.temporal.io/sdk/client" 19 | ) 20 | 21 | type OpenCloser interface { 22 | Open() error 23 | Close() error 24 | } 25 | 26 | // Main represents the application. 27 | type Main struct { 28 | db OpenCloser 29 | httpServer OpenCloser 30 | scheduler OpenCloser 31 | worker OpenCloser 32 | client client.Client 33 | } 34 | 35 | // NewMain returns a new instance of Main. 36 | func NewMain() *Main { 37 | // TODO: NewMain() should accept a config parameter 38 | // which contains all the configuration that has been 39 | // parsed from a TOML file. 40 | db := postgres.NewDB("postgres://postgres:password@postgresql:5432/postgres", true) 41 | 42 | dbService := postgres.NewDBService(db) 43 | messageService := jsonschema.NewMessageService() 44 | artifactService := filesystem.NewArtifactService() 45 | commandService := docker.NewCommandService() 46 | worker := temporal.NewWorker() 47 | scheduler := scheduler.NewScheduler() 48 | logger := zap.NewLogger() 49 | httpServer := http.NewServer(":5000") 50 | 51 | app := &cosmos.App{ 52 | DBService: dbService, 53 | CommandService: commandService, 54 | MessageService: messageService, 55 | ArtifactService: artifactService, 56 | SchedulerService: scheduler, 57 | WorkerService: worker, 58 | Logger: logger, 59 | } 60 | commandService.App = app 61 | worker.App = app 62 | scheduler.App = app 63 | httpServer.App = app 64 | 65 | client, err := client.NewClient(client.Options{HostPort: "temporal:7233", Logger: logger}) 66 | if err != nil { 67 | log.Fatal("Unable to create temporal client. err: " + err.Error()) 68 | } 69 | worker.Client = client 70 | 71 | return &Main{ 72 | db: db, 73 | httpServer: httpServer, 74 | scheduler: scheduler, 75 | worker: worker, 76 | client: client, 77 | } 78 | } 79 | 80 | func (m *Main) startup() error { 81 | if err := m.db.Open(); err != nil { 82 | return fmt.Errorf("cannot open db: %w", err) 83 | } 84 | if err := m.httpServer.Open(); err != nil { 85 | return fmt.Errorf("cannot start http server: %w", err) 86 | } 87 | if err := m.worker.Open(); err != nil { 88 | return fmt.Errorf("cannot start worker: %w", err) 89 | } 90 | if err := m.scheduler.Open(); err != nil { 91 | return fmt.Errorf("cannot start scheduler: %w", err) 92 | } 93 | 94 | fmt.Printf(` 95 | 96 | _________ 97 | \_ ___ \ ____ ______ _____ ____ ______ 98 | / \ \/ / _ \ / ___/ / \ / _ \ / ___/ 99 | \ \____( <_> ) \___ \ | Y Y \( <_> ) \___ \ 100 | \______ / \____/ /____ >|__|_| / \____/ /____ > 101 | \/ \/ \/ \/ 102 | 103 | is now accepting connections at %s 104 | 105 | 106 | `, "http://localhost:5000") 107 | 108 | return nil 109 | } 110 | 111 | func (m *Main) shutdown() error { 112 | if err := m.scheduler.Close(); err != nil { 113 | return err 114 | } 115 | if err := m.worker.Close(); err != nil { 116 | return err 117 | } 118 | if err := m.httpServer.Close(); err != nil { 119 | return err 120 | } 121 | if err := m.db.Close(); err != nil { 122 | return err 123 | } 124 | m.client.Close() 125 | return nil 126 | } 127 | 128 | func main() { 129 | // Setup SIGINT (Ctrl-C) handler. 130 | interruptChannel := make(chan os.Signal, 1) 131 | signal.Notify(interruptChannel, os.Interrupt) 132 | 133 | m := NewMain() 134 | 135 | // Start the application. 136 | if err := m.startup(); err != nil { 137 | m.shutdown() 138 | log.Fatalf("cosmos: application startup failed: %s", err) 139 | } 140 | 141 | // Wait for Ctrl-C. 142 | <-interruptChannel 143 | 144 | // Shutdown the application. 145 | if err := m.shutdown(); err != nil { 146 | log.Fatalf("cosmos: application shutdown failed: %s", err) 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /cosmos-frontend/src/views/ConnectorsByType.vue: -------------------------------------------------------------------------------- 1 | 45 | 46 | 130 | -------------------------------------------------------------------------------- /cosmos-backend/connector.go: -------------------------------------------------------------------------------- 1 | package cosmos 2 | 3 | import ( 4 | "context" 5 | "time" 6 | ) 7 | 8 | // Connector types. 9 | const ( 10 | ConnectorTypeSource = "source" 11 | ConnectorTypeDestination = "destination" 12 | ) 13 | 14 | var DestinationTypes = []string{ 15 | "postgres", 16 | "bigquery", 17 | "redshift", 18 | "snowflake", 19 | "mysql", 20 | "other", 21 | } 22 | 23 | // Connector represents a source or destination connector. 24 | type Connector struct { 25 | ID int `json:"id"` 26 | Name string `json:"name"` 27 | Type string `json:"type"` 28 | DockerImageName string `json:"dockerImageName"` 29 | DockerImageTag string `json:"dockerImageTag"` 30 | DestinationType string `json:"destinationType"` 31 | Spec Message `json:"spec"` 32 | CreatedAt time.Time `json:"createdAt"` 33 | UpdatedAt time.Time `json:"updatedAt"` 34 | } 35 | 36 | func (c *Connector) HasValidDestinationType() bool { 37 | switch c.Type { 38 | case ConnectorTypeSource: 39 | if c.DestinationType == "" { 40 | return true 41 | } 42 | case ConnectorTypeDestination: 43 | for _, t := range DestinationTypes { 44 | if t == c.DestinationType { 45 | return true 46 | } 47 | } 48 | } 49 | return false 50 | } 51 | 52 | // Validate performs some basic validation on the connector object during create and update. 53 | func (c *Connector) Validate() error { 54 | if c.Name == "" { 55 | return Errorf(EINVALID, "Connector name required") 56 | } else if c.Type != ConnectorTypeSource && c.Type != ConnectorTypeDestination { 57 | return Errorf(EINVALID, "Connector type must be one of 'source' or 'destination'") 58 | } else if c.DockerImageName == "" || c.DockerImageTag == "" { 59 | return Errorf(EINVALID, "Docker image name and tag are required") 60 | } else if !c.HasValidDestinationType() { 61 | return Errorf(EINVALID, "Invalid destination type") 62 | } 63 | return nil 64 | } 65 | 66 | // ConnectorFilter represents a connector search filter. 67 | type ConnectorFilter struct { 68 | ID *int `json:"id"` 69 | Name *string `json:"name"` 70 | Type *string `json:"type"` 71 | 72 | Offset int `json:"offset"` 73 | Limit int `json:"limit"` 74 | } 75 | 76 | // ConnectorUpdate represent connector fields that can be updated. 77 | type ConnectorUpdate struct { 78 | Name *string `json:"name"` 79 | DockerImageName *string `json:"dockerImageName"` 80 | DockerImageTag *string `json:"dockerImageTag"` 81 | DestinationType *string `json:"destinationType"` 82 | } 83 | 84 | type ConnectorService interface { 85 | FindConnectorByID(ctx context.Context, id int) (*Connector, error) 86 | FindConnectors(ctx context.Context, filter ConnectorFilter) ([]*Connector, int, error) 87 | CreateConnector(ctx context.Context, connector *Connector) error 88 | UpdateConnector(ctx context.Context, id int, connector *Connector) error 89 | DeleteConnector(ctx context.Context, id int) error 90 | } 91 | 92 | func (a *App) CreateConnector(ctx context.Context, connector *Connector) error { 93 | // Perform basic field validation. 94 | if err := connector.Validate(); err != nil { 95 | return err 96 | } 97 | 98 | // Get connections specification for the connector and update it in the connector object. 99 | msg, err := a.Spec(ctx, connector) 100 | if err != nil { 101 | return err 102 | } 103 | connector.Spec = *msg 104 | 105 | return a.DBService.CreateConnector(ctx, connector) 106 | } 107 | 108 | func (a *App) UpdateConnector(ctx context.Context, id int, upd *ConnectorUpdate) (*Connector, error) { 109 | // Fetch the current connector object from the database. 110 | connector, err := a.FindConnectorByID(ctx, id) 111 | if err != nil { 112 | return nil, err 113 | } 114 | 115 | // Update fields if set. 116 | if v := upd.Name; v != nil { 117 | connector.Name = *v 118 | } 119 | if v := upd.DockerImageName; v != nil { 120 | connector.DockerImageName = *v 121 | } 122 | if v := upd.DockerImageTag; v != nil { 123 | connector.DockerImageTag = *v 124 | } 125 | if v := upd.DestinationType; v != nil { 126 | connector.DestinationType = *v 127 | } 128 | 129 | // Perform basic validation to make sure that the updates are correct. 130 | if err := connector.Validate(); err != nil { 131 | return nil, err 132 | } 133 | 134 | // Get connections specification for the connector and update it in the connector object. 135 | msg, err := a.Spec(ctx, connector) 136 | if err != nil { 137 | return nil, err 138 | } 139 | connector.Spec = *msg 140 | 141 | if err := a.DBService.UpdateConnector(ctx, id, connector); err != nil { 142 | return nil, err 143 | } 144 | 145 | return connector, nil 146 | } 147 | -------------------------------------------------------------------------------- /cosmos-backend/http/connector.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "cosmos" 5 | "net/http" 6 | "strconv" 7 | 8 | "github.com/gorilla/mux" 9 | ) 10 | 11 | func (s *Server) registerConnectorRoutes(r *mux.Router) { 12 | r.HandleFunc("/connectors", s.findConnectors).Methods("GET") 13 | r.HandleFunc("/connectors", s.createConnector).Methods("POST") 14 | r.HandleFunc("/connectors/{id}", s.updateConnector).Methods("PATCH") 15 | r.HandleFunc("/connectors/{id}", s.deleteConnector).Methods("DELETE") 16 | 17 | r.HandleFunc("/connectors/{id}/connection-spec-form", s.connectionSpecForm).Methods("GET") 18 | r.HandleFunc("/connectors/destination-types", s.getDestinationTypes).Methods("GET") 19 | } 20 | 21 | func (s *Server) findConnectors(w http.ResponseWriter, r *http.Request) { 22 | // Get the "type" if any from the query params. 23 | filter := cosmos.ConnectorFilter{} 24 | if connectorType, ok := r.URL.Query()["type"]; ok { 25 | filter.Type = &connectorType[0] 26 | } 27 | 28 | connectors, totalConnectors, err := s.App.FindConnectors(r.Context(), filter) 29 | if err != nil { 30 | s.ReplyWithSanitizedError(w, r, err) 31 | return 32 | } 33 | 34 | ret := map[string]interface{}{ 35 | "connectors": connectors, 36 | "totalConnectors": totalConnectors, 37 | } 38 | 39 | w.Header().Set("Content-Type", "application/json") 40 | if err := json.NewEncoder(w).Encode(&ret); err != nil { 41 | s.LogError(r, err) 42 | } 43 | } 44 | 45 | func (s *Server) createConnector(w http.ResponseWriter, r *http.Request) { 46 | var connector cosmos.Connector 47 | if err := json.NewDecoder(r.Body).Decode(&connector); err != nil { 48 | s.ReplyWithSanitizedError(w, r, cosmos.Errorf(cosmos.EINVALID, "Invalid JSON body")) 49 | return 50 | } 51 | 52 | err := s.App.CreateConnector(r.Context(), &connector) 53 | if err != nil { 54 | s.ReplyWithSanitizedError(w, r, err) 55 | return 56 | } 57 | 58 | w.Header().Set("Content-Type", "application/json") 59 | w.WriteHeader(http.StatusCreated) 60 | if err := json.NewEncoder(w).Encode(&connector); err != nil { 61 | s.LogError(r, err) 62 | } 63 | } 64 | 65 | func (s *Server) updateConnector(w http.ResponseWriter, r *http.Request) { 66 | id, err := strconv.Atoi(mux.Vars(r)["id"]) 67 | if err != nil { 68 | s.ReplyWithSanitizedError(w, r, cosmos.Errorf(cosmos.EINVALID, "Invalid connector ID")) 69 | return 70 | } 71 | 72 | upd := &cosmos.ConnectorUpdate{} 73 | if err := json.NewDecoder(r.Body).Decode(upd); err != nil { 74 | s.ReplyWithSanitizedError(w, r, cosmos.Errorf(cosmos.EINVALID, "Invalid JSON body")) 75 | return 76 | } 77 | 78 | connector, err := s.App.UpdateConnector(r.Context(), id, upd) 79 | if err != nil { 80 | s.ReplyWithSanitizedError(w, r, err) 81 | return 82 | } 83 | 84 | w.Header().Set("Content-Type", "application/json") 85 | if err := json.NewEncoder(w).Encode(connector); err != nil { 86 | s.LogError(r, err) 87 | } 88 | } 89 | 90 | func (s *Server) deleteConnector(w http.ResponseWriter, r *http.Request) { 91 | // Parse connector id from path params. 92 | id, err := strconv.Atoi(mux.Vars(r)["id"]) 93 | if err != nil { 94 | s.ReplyWithSanitizedError(w, r, cosmos.Errorf(cosmos.EINVALID, "Invalid connector ID")) 95 | return 96 | } 97 | 98 | if err := s.App.DeleteConnector(r.Context(), id); err != nil { 99 | s.ReplyWithSanitizedError(w, r, err) 100 | } 101 | 102 | w.Header().Set("Content-Type", "application/json") 103 | w.Write([]byte(`{}`)) 104 | } 105 | 106 | func (s *Server) connectionSpecForm(w http.ResponseWriter, r *http.Request) { 107 | id, err := strconv.Atoi(mux.Vars(r)["id"]) 108 | if err != nil { 109 | s.ReplyWithSanitizedError(w, r, cosmos.Errorf(cosmos.EINVALID, "Invalid connector ID")) 110 | return 111 | } 112 | 113 | connector, err := s.App.FindConnectorByID(r.Context(), id) 114 | if err != nil { 115 | s.ReplyWithSanitizedError(w, r, err) 116 | return 117 | } 118 | 119 | // If the connection spec is nil, trigger a connector update which will fetch the connection spec. 120 | if connector.Spec.Spec == nil { 121 | var err error 122 | connector, err = s.App.UpdateConnector(r.Context(), id, &cosmos.ConnectorUpdate{}) 123 | if err != nil { 124 | s.ReplyWithSanitizedError(w, r, err) 125 | return 126 | } 127 | } 128 | 129 | createForm := s.App.MessageToForm(r.Context(), &connector.Spec, nil) 130 | 131 | w.Header().Set("Content-Type", "application/json") 132 | if err := json.NewEncoder(w).Encode(createForm); err != nil { 133 | s.LogError(r, err) 134 | } 135 | } 136 | 137 | func (s *Server) getDestinationTypes(w http.ResponseWriter, r *http.Request) { 138 | w.Header().Set("Content-Type", "application/json") 139 | if err := json.NewEncoder(w).Encode(cosmos.DestinationTypes); err != nil { 140 | s.LogError(r, err) 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /cosmos-backend/http/sync.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "cosmos" 5 | "net/http" 6 | "strconv" 7 | 8 | "github.com/gorilla/mux" 9 | ) 10 | 11 | func (s *Server) registerSyncRoutes(r *mux.Router) { 12 | r.HandleFunc("/syncs", s.findSyncs).Methods("GET") 13 | r.HandleFunc("/syncs/{id}", s.findSyncs).Methods("GET") 14 | r.HandleFunc("/syncs", s.createSync).Methods("POST") 15 | r.HandleFunc("/syncs/{id}", s.updateSync).Methods("PATCH") 16 | r.HandleFunc("/syncs/{id}", s.deleteSync).Methods("DELETE") 17 | 18 | r.HandleFunc("/syncs/{id}/edit-form", s.editSyncForm).Methods("GET") 19 | r.HandleFunc("/syncs/{id}/sync-now", s.syncNow).Methods("POST") 20 | } 21 | 22 | func (s *Server) findSyncs(w http.ResponseWriter, r *http.Request) { 23 | filter := cosmos.SyncFilter{} 24 | 25 | if v, ok := mux.Vars(r)["id"]; ok { 26 | syncID, err := strconv.Atoi(v) 27 | if err != nil { 28 | s.ReplyWithSanitizedError(w, r, cosmos.Errorf(cosmos.EINVALID, "Invalid sync ID")) 29 | return 30 | } 31 | filter.ID = &syncID 32 | } 33 | 34 | syncs, totalSyncs, err := s.App.FindSyncs(r.Context(), filter) 35 | if err != nil { 36 | s.ReplyWithSanitizedError(w, r, err) 37 | return 38 | } 39 | 40 | ret := map[string]interface{}{ 41 | "syncs": syncs, 42 | "totalSyncs": totalSyncs, 43 | } 44 | 45 | w.Header().Set("Content-Type", "application/json") 46 | if err := json.NewEncoder(w).Encode(&ret); err != nil { 47 | s.LogError(r, err) 48 | } 49 | } 50 | 51 | func (s *Server) createSync(w http.ResponseWriter, r *http.Request) { 52 | var sync cosmos.Sync 53 | if err := json.NewDecoder(r.Body).Decode(&sync); err != nil { 54 | s.ReplyWithSanitizedError(w, r, cosmos.Errorf(cosmos.EINVALID, "Invalid JSON body")) 55 | return 56 | } 57 | 58 | err := s.App.CreateSync(r.Context(), &sync) 59 | if err != nil { 60 | s.ReplyWithSanitizedError(w, r, err) 61 | return 62 | } 63 | 64 | w.Header().Set("Content-Type", "application/json") 65 | w.WriteHeader(http.StatusCreated) 66 | if err := json.NewEncoder(w).Encode(&sync); err != nil { 67 | s.LogError(r, err) 68 | } 69 | } 70 | 71 | func (s *Server) updateSync(w http.ResponseWriter, r *http.Request) { 72 | id, err := strconv.Atoi(mux.Vars(r)["id"]) 73 | if err != nil { 74 | s.ReplyWithSanitizedError(w, r, cosmos.Errorf(cosmos.EINVALID, "Invalid sync ID")) 75 | return 76 | } 77 | 78 | upd := &cosmos.SyncUpdate{} 79 | if err = json.NewDecoder(r.Body).Decode(upd); err != nil { 80 | s.ReplyWithSanitizedError(w, r, cosmos.Errorf(cosmos.EINVALID, "Invalid JSON body")) 81 | return 82 | } 83 | 84 | sync, err := s.App.UpdateSync(r.Context(), id, upd) 85 | if err != nil { 86 | s.ReplyWithSanitizedError(w, r, err) 87 | return 88 | } 89 | 90 | w.Header().Set("Content-Type", "application/json") 91 | if err := json.NewEncoder(w).Encode(sync); err != nil { 92 | s.LogError(r, err) 93 | } 94 | } 95 | 96 | func (s *Server) deleteSync(w http.ResponseWriter, r *http.Request) { 97 | // Parse sync id from path params. 98 | id, err := strconv.Atoi(mux.Vars(r)["id"]) 99 | if err != nil { 100 | s.ReplyWithSanitizedError(w, r, cosmos.Errorf(cosmos.EINVALID, "Invalid sync ID")) 101 | return 102 | } 103 | 104 | if err := s.App.DeleteSync(r.Context(), id); err != nil { 105 | s.ReplyWithSanitizedError(w, r, err) 106 | } 107 | 108 | w.Header().Set("Content-Type", "application/json") 109 | w.Write([]byte(`{}`)) 110 | } 111 | 112 | func (s *Server) editSyncForm(w http.ResponseWriter, r *http.Request) { 113 | id, err := strconv.Atoi(mux.Vars(r)["id"]) 114 | if err != nil { 115 | s.ReplyWithSanitizedError(w, r, cosmos.Errorf(cosmos.EINVALID, "Invalid sync ID")) 116 | return 117 | } 118 | 119 | sync, err := s.App.FindSyncByID(r.Context(), id) 120 | if err != nil { 121 | s.ReplyWithSanitizedError(w, r, err) 122 | return 123 | } 124 | 125 | baseForm := s.App.MessageToForm( 126 | r.Context(), 127 | &sync.SourceEndpoint.Catalog, 128 | sync.DestinationEndpoint.Connector.Spec.Spec.SupportedDestinationSyncModes, 129 | ) 130 | baseForm.Merge(&sync.Config) 131 | 132 | w.Header().Set("Content-Type", "application/json") 133 | if err := json.NewEncoder(w).Encode(baseForm); err != nil { 134 | s.LogError(r, err) 135 | } 136 | } 137 | 138 | func (s *Server) syncNow(w http.ResponseWriter, r *http.Request) { 139 | syncID, err := strconv.Atoi(mux.Vars(r)["id"]) 140 | if err != nil { 141 | s.ReplyWithSanitizedError(w, r, cosmos.Errorf(cosmos.EINVALID, "Invalid sync ID")) 142 | return 143 | } 144 | 145 | runOptions := &cosmos.RunOptions{} 146 | if err := json.NewDecoder(r.Body).Decode(runOptions); err != nil { 147 | s.ReplyWithSanitizedError(w, r, cosmos.Errorf(cosmos.EINVALID, "Invalid run options")) 148 | return 149 | } 150 | 151 | if err := s.App.Schedule(&syncID, runOptions); err != nil { 152 | s.ReplyWithSanitizedError(w, r, err) 153 | return 154 | } 155 | } 156 | -------------------------------------------------------------------------------- /cosmos-backend/endpoint.go: -------------------------------------------------------------------------------- 1 | package cosmos 2 | 3 | import ( 4 | "context" 5 | "time" 6 | ) 7 | 8 | // EndPoint represents a "Connector" that has been configured for a particular endpoint. 9 | type Endpoint struct { 10 | ID int `json:"id"` 11 | Name string `json:"name"` 12 | Type string `json:"type"` 13 | ConnectorID int `json:"connectorID"` 14 | Config Form `json:"config"` 15 | Catalog Message `json:"catalog"` 16 | LastDiscovered time.Time `json:"lastDiscovered"` 17 | CreatedAt time.Time `json:"createdAt"` 18 | UpdatedAt time.Time `json:"updatedAt"` 19 | Connector *Connector `json:"connector"` 20 | } 21 | 22 | func (e *Endpoint) Validate() error { 23 | if e.Name == "" { 24 | return Errorf(EINVALID, "Endpoint name required") 25 | } else if e.Type != ConnectorTypeSource && e.Type != ConnectorTypeDestination { 26 | return Errorf(EINVALID, "Endpoint type must be one of 'source' or 'destination'") 27 | } else if e.ConnectorID == 0 { 28 | return Errorf(EINVALID, "A connector must be selected") 29 | } 30 | return nil 31 | } 32 | 33 | type EndpointUpdate struct { 34 | Name *string `json:"name"` 35 | Config *Form `json:"config"` 36 | } 37 | 38 | type EndpointFilter struct { 39 | ID *int `json:"id"` 40 | Name *string `json:"name"` 41 | Type *string `json:"type"` 42 | 43 | Offset int `json:"offset"` 44 | Limit int `json:"limit"` 45 | } 46 | 47 | type EndpointService interface { 48 | FindEndpointByID(ctx context.Context, id int) (*Endpoint, error) 49 | FindEndpoints(ctx context.Context, filter EndpointFilter) ([]*Endpoint, int, error) 50 | CreateEndpoint(ctx context.Context, endpoint *Endpoint) error 51 | UpdateEndpoint(ctx context.Context, id int, endpoint *Endpoint) error 52 | DeleteEndpoint(ctx context.Context, id int) error 53 | } 54 | 55 | func (a *App) CreateEndpoint(ctx context.Context, endpoint *Endpoint) error { 56 | // Perform basic field validation. 57 | if err := endpoint.Validate(); err != nil { 58 | return err 59 | } 60 | 61 | connector, err := a.FindConnectorByID(ctx, endpoint.ConnectorID) 62 | if err != nil { 63 | return err 64 | } 65 | 66 | config := endpoint.Config.ToSpec() 67 | 68 | if err := a.Validate(ctx, config, &connector.Spec); err != nil { 69 | return err 70 | } 71 | 72 | msg, err := a.Check(ctx, connector, config) 73 | if err != nil { 74 | return err 75 | } 76 | 77 | if msg.ConnectionStatus.Status != ConnectionStatusSucceeded { 78 | var connectionError string 79 | if msg.ConnectionStatus.Message != nil { 80 | connectionError = *msg.ConnectionStatus.Message 81 | } 82 | return Errorf(EINVALID, "The configuration provided is invalid. %s", connectionError) 83 | } 84 | 85 | msg, err = a.Discover(ctx, connector, config) 86 | if err != nil { 87 | return err 88 | } 89 | endpoint.Catalog = *msg 90 | 91 | endpoint.Connector = connector 92 | 93 | return a.DBService.CreateEndpoint(ctx, endpoint) 94 | } 95 | 96 | func (a *App) UpdateEndpoint(ctx context.Context, id int, upd *EndpointUpdate) (*Endpoint, error) { 97 | // Fetch the current endpoint object from the database. 98 | endpoint, err := a.FindEndpointByID(ctx, id) 99 | if err != nil { 100 | return nil, err 101 | } 102 | 103 | // Update fields if set. 104 | if v := upd.Name; v != nil { 105 | endpoint.Name = *v 106 | } 107 | if v := upd.Config; v != nil { 108 | endpoint.Config = *v 109 | } 110 | 111 | // Perform basic validation to make sure that the updates are correct. 112 | if err := endpoint.Validate(); err != nil { 113 | return nil, err 114 | } 115 | 116 | config := endpoint.Config.ToSpec() 117 | 118 | if err := a.Validate(ctx, config, &endpoint.Connector.Spec); err != nil { 119 | return nil, err 120 | } 121 | 122 | msg, err := a.Check(ctx, endpoint.Connector, config) 123 | if err != nil { 124 | return nil, err 125 | } 126 | 127 | if msg.ConnectionStatus.Status != ConnectionStatusSucceeded { 128 | var connectionError string 129 | if msg.ConnectionStatus.Message != nil { 130 | connectionError = *msg.ConnectionStatus.Message 131 | } 132 | return nil, Errorf(EINVALID, "The configuration provided is invalid. %s", connectionError) 133 | } 134 | 135 | if err := a.DBService.UpdateEndpoint(ctx, id, endpoint); err != nil { 136 | return nil, err 137 | } 138 | 139 | return endpoint, nil 140 | } 141 | 142 | func (a *App) RediscoverEndpoint(ctx context.Context, id int) error { 143 | // Fetch the current endpoint object from the database. 144 | endpoint, err := a.FindEndpointByID(ctx, id) 145 | if err != nil { 146 | return err 147 | } 148 | 149 | config := endpoint.Config.ToSpec() 150 | 151 | msg, err := a.Discover(ctx, endpoint.Connector, config) 152 | if err != nil { 153 | return err 154 | } 155 | endpoint.Catalog = *msg 156 | 157 | endpoint.LastDiscovered = time.Now() 158 | 159 | if err := a.DBService.UpdateEndpoint(ctx, id, endpoint); err != nil { 160 | return err 161 | } 162 | 163 | return nil 164 | } 165 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # COSMOS 2 | 3 | An [Airbyte](https://github.com/airbytehq/airbyte) clone written in Go and Vue.js mainly as a hobby project. 4 | 5 | Works with all Airbyte connectors (source and destination) out of the box. 6 | 7 | Features a simpler and cleaner UI (see screenshots below). 8 | 9 | ## Why? 10 | 11 | I started working on an Airbyte clone primarily to learn Go and Vue.js. 12 | 13 | Along the way, I added a few nice touches to the UI. 14 | 15 | ## Tech stack 16 | 17 | **Backend**: Golang 18 | 19 | **Frontend**: [Vue](https://vuejs.org/) + [Vuetify](https://vuetifyjs.com/en/) (material design) 20 | 21 | **Workflow orchestration**: [Temporal](https://temporal.io/) 22 | 23 | **Realtime UI updates**: [Supabase Realtime](https://github.com/supabase/realtime) (websockets) 24 | 25 | ## Concepts 26 | 27 | | Name | Description | 28 | | ---- | ----------- | 29 | | Connectors | These are the usual Airbyte [connectors](https://docs.airbyte.io/integrations). | 30 | | Endpoints | Actual physical endpoints in your origanization. For example, a PostgreSQL server running at X.X.X.X:5432 is one *endpoint* while a PostgreSQL server running at Y.Y.Y.Y:5432 is a second *endpoint* although both endpoints make use of the Postgres *connector*. | 31 | | Syncs | A *sync* is nothing more than a connection between two *endpoints* describing the way that data should be replicated between those two *endpoints*. | 32 | | Runs | A *run* is an instance of a *sync* with a unique execution date. | 33 | 34 | ## Getting started 35 | 36 | Run the application with docker-compose. 37 | 38 | $ git clone https://github.com/varunbpatil/cosmos.git 39 | $ cd cosmos 40 | $ docker-compose up 41 | 42 | Wait for the the message "Accepting connections at http://localhost:5000" on the console. 43 | 44 | The application is now ready to be used. 45 | 46 | ## Screenshot tour 47 | 48 | The *Connectors* page comes pre-populated with all of Airbyte's source and destination connectors. 49 | 50 | ![Connectors](images/connectors.png) 51 | 52 | You also have the ability to add new connectors, delete unused connectors and 53 | modify existing connectors with your own docker images and versions. 54 | 55 | ![Connectors Edit](images/connectors_edit.png) 56 | 57 | You start by creating source and destination *Endpoints*. You get fuzzy text 58 | search in all dropdown boxes. 59 | 60 | ![Connectors Edit](images/endpoints_create.png) 61 | 62 | The page should be automatically updated immediately after the *endpoint* is created. 63 | 64 | ![Connectors Edit](images/endpoints.png) 65 | 66 | The next step is to connect a source and destination *endpoint* pair with a *Sync*. 67 | 68 | ![Connectors Edit](images/syncs_create.png) 69 | 70 | The page should automatically update immediately after the *Sync* is created. You can 71 | click on the **Enable Sync** toggle button to enable the sync to run on a schedule or 72 | click the **Sync Now** button to start an ad-hoc sync. 73 | 74 | ![Connectors Edit](images/syncs.png) 75 | 76 | Clicking on a sync will show you all the *runs* for that *sync*. 77 | 78 | All *syncs* and their corresponding *runs* have a colored bar on the left to indicate completion status. 79 | 80 | You can then filter *runs* based on their status or execution date. 81 | 82 | You can also cancel a run in-between. 83 | 84 | You also get to see a near-realtime update to the number of records being replicated. 85 | 86 | ![Connectors Edit](images/runs.png) 87 | 88 | Clicking on a particular *run* will show you the logs and other details for 89 | that particular *run* in near-realtime. 90 | 91 | ![Connectors Edit](images/logs.png) 92 | 93 | You can also edit/delete *Endpoints* and *Syncs* (just like you can 94 | *Connectors*) at any time by clicking on the **Edit Sync** or **Edit Endpoint** 95 | buttons. 96 | 97 | ![Connectors Edit](images/syncs_edit.png) 98 | 99 | You can also clear state (for incremental syncs) and/or completely clear all 100 | data on the destination by clicking on **Edit Sync** and then clicking on the 101 | Kebab menu button (three vertical dots) on the bottom left of the dialog. 102 | 103 | ![Connectors Edit](images/clear_state.png) 104 | 105 | ## License 106 | MIT 107 | 108 | ## Thanks 109 | Thanks to the Airbyte team and community for the awesome software and 110 | connectors. What I've written is not a new product in any way, just a 111 | reimagination. 112 | -------------------------------------------------------------------------------- /cosmos-frontend/src/components/CreateConnector.vue: -------------------------------------------------------------------------------- 1 | 70 | 71 | 157 | -------------------------------------------------------------------------------- /cosmos-backend/message.go: -------------------------------------------------------------------------------- 1 | package cosmos 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/iancoleman/orderedmap" 8 | ) 9 | 10 | const ( 11 | MessageTypeRecord = "RECORD" 12 | MessageTypeState = "STATE" 13 | MessageTypeLog = "LOG" 14 | MessageTypeSpec = "SPEC" 15 | MessageTypeConnectionStatus = "CONNECTION_STATUS" 16 | MessageTypeCatalog = "CATALOG" 17 | MessageTypeConfiguredCatalog = "CONFIGURED_CATALOG" 18 | 19 | ConnectionStatusSucceeded = "SUCCEEDED" 20 | ConnectionStatusFailed = "FAILED" 21 | 22 | SyncModeFullRefresh = "full_refresh" 23 | SyncModeIncremental = "incremental" 24 | 25 | DestinationSyncModeAppend = "append" 26 | DestinationSyncModeOverwrite = "overwrite" 27 | DestinationSyncModeAppendDedup = "append_dedup" 28 | DestinationSyncModeUpsertDedup = "upsert_dedup" 29 | 30 | LogLevelFatal = "FATAL" 31 | LogLevelError = "ERROR" 32 | LogLevelWarn = "WARN" 33 | LogLevelInfo = "INFO" 34 | LogLevelDebug = "DEBUG" 35 | LogLevelTrace = "TRACE" 36 | ) 37 | 38 | type Message struct { 39 | Type string `json:"type,omitempty"` 40 | Log *Log `json:"log,omitempty"` 41 | Spec *Spec `json:"spec,omitempty"` 42 | ConnectionStatus *ConnectionStatus `json:"connectionStatus,omitempty"` 43 | Catalog *Catalog `json:"catalog,omitempty"` 44 | ConfiguredCatalog *ConfiguredCatalog `json:"configuredCatalog,omitempty"` 45 | Record *Record `json:"record,omitempty"` 46 | State *State `json:"state,omitempty"` 47 | } 48 | 49 | type Log struct { 50 | Level string `json:"level,omitempty"` 51 | Message string `json:"message,omitempty"` 52 | } 53 | 54 | func (l *Log) String() string { 55 | // See https://www.lihaoyi.com/post/BuildyourownCommandLinewithANSIescapecodes.html for ansi colors. 56 | colorPrefix := "" 57 | colorReset := "\u001b[0m" 58 | 59 | switch l.Level { 60 | case LogLevelFatal, LogLevelError: 61 | colorPrefix = "\u001b[31m" 62 | case LogLevelWarn: 63 | colorPrefix = "\u001b[33m" 64 | case LogLevelInfo: 65 | colorPrefix = "\u001b[32m" 66 | case LogLevelDebug, LogLevelTrace: 67 | colorPrefix = "\u001b[34m" 68 | default: 69 | panic("Unknown log level") 70 | } 71 | 72 | return fmt.Sprintf("%s%s%s %s", colorPrefix, l.Level, colorReset, l.Message) 73 | } 74 | 75 | type Spec struct { 76 | ConnectionSpecification orderedmap.OrderedMap `json:"connectionSpecification,omitempty"` 77 | DocumentationURL string `json:"documentationUrl,omitempty"` 78 | ChangelogURL string `json:"changelogUrl,omitempty"` 79 | SupportsIncremental bool `json:"supportsIncremental,omitempty"` 80 | SupportsNormalization bool `json:"supportsNormalization,omitempty"` 81 | SupportsDBT bool `json:"supportsDBT,omitempty"` 82 | SupportedDestinationSyncModes []string `json:"supported_destination_sync_modes,omitempty"` 83 | } 84 | 85 | type Catalog struct { 86 | Streams []Stream `json:"streams,omitempty"` 87 | } 88 | 89 | type Stream struct { 90 | Name string `json:"name,omitempty"` 91 | JSONSchema orderedmap.OrderedMap `json:"json_schema,omitempty"` 92 | SupportedSyncModes []string `json:"supported_sync_modes,omitempty"` 93 | SourceDefinedCursor bool `json:"source_defined_cursor,omitempty"` 94 | DefaultCursorField []string `json:"default_cursor_field,omitempty"` 95 | SourceDefinedPrimaryKey [][]string `json:"source_defined_primary_key,omitempty"` 96 | Namespace *string `json:"namespace,omitempty"` 97 | } 98 | 99 | type ConfiguredCatalog struct { 100 | Streams []ConfiguredStream `json:"streams,omitempty"` 101 | } 102 | 103 | type ConfiguredStream struct { 104 | Stream Stream `json:"stream,omitempty"` 105 | SyncMode *string `json:"sync_mode,omitempty"` 106 | CursorField []string `json:"cursor_field,omitempty"` 107 | DestinationSyncMode *string `json:"destination_sync_mode,omitempty"` 108 | PrimaryKey [][]string `json:"primary_key,omitempty"` 109 | } 110 | 111 | type Record struct { 112 | Stream string `json:"stream,omitempty"` 113 | Data map[string]interface{} `json:"data,omitempty"` 114 | EmittedAt int `json:"emitted_at,omitempty"` 115 | Namespace *string `json:"namespace,omitempty"` 116 | } 117 | 118 | type State struct { 119 | Data map[string]interface{} `json:"data,omitempty"` 120 | } 121 | 122 | type ConnectionStatus struct { 123 | Status string `json:"status,omitempty"` 124 | Message *string `json:"message,omitempty"` 125 | } 126 | 127 | type MessageService interface { 128 | CreateMessage(ctx context.Context, raw []byte) (*Message, error) 129 | MessageToForm(ctx context.Context, message *Message, additionalInfo interface{}) *Form 130 | Validate(ctx context.Context, raw interface{}, message *Message) error 131 | } 132 | 133 | func (m *Message) String() string { 134 | b, _ := json.Marshal(m) 135 | return string(b) 136 | } 137 | 138 | func (s *Stream) IsSyncModeAvailable(syncMode string) bool { 139 | // SyncModeFullRefresh is supported by all sources even if sync.SupportedSyncModes is empty. 140 | if syncMode == SyncModeFullRefresh { 141 | return true 142 | } 143 | for _, s := range s.SupportedSyncModes { 144 | if s == syncMode { 145 | return true 146 | } 147 | } 148 | return false 149 | } 150 | -------------------------------------------------------------------------------- /cosmos-backend/http/endpoint.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "cosmos" 5 | "net/http" 6 | "strconv" 7 | 8 | "github.com/gorilla/mux" 9 | ) 10 | 11 | func (s *Server) registerEndpointRoutes(r *mux.Router) { 12 | r.HandleFunc("/endpoints", s.findEndpoints).Methods("GET") 13 | r.HandleFunc("/endpoints", s.createEndpoint).Methods("POST") 14 | r.HandleFunc("/endpoints/{id}", s.updateEndpoint).Methods("PATCH") 15 | r.HandleFunc("/endpoints/{id}", s.deleteEndpoint).Methods("DELETE") 16 | 17 | r.HandleFunc("/endpoints/{id}/edit-form", s.editEndpointForm).Methods("GET") 18 | r.HandleFunc("/endpoints/{id}/rediscover", s.rediscoverEndpoint).Methods("POST") 19 | r.HandleFunc("/endpoints/{srcID}/{dstID}/catalog-form", s.catalogForm).Methods("GET") 20 | } 21 | 22 | func (s *Server) findEndpoints(w http.ResponseWriter, r *http.Request) { 23 | // Get the "type" if any from the query params. 24 | filter := cosmos.EndpointFilter{} 25 | if endpointType, ok := r.URL.Query()["type"]; ok { 26 | filter.Type = &endpointType[0] 27 | } 28 | 29 | endpoints, totalEndpoints, err := s.App.FindEndpoints(r.Context(), filter) 30 | if err != nil { 31 | s.ReplyWithSanitizedError(w, r, err) 32 | return 33 | } 34 | 35 | ret := map[string]interface{}{ 36 | "endpoints": endpoints, 37 | "totalEndpoints": totalEndpoints, 38 | } 39 | 40 | w.Header().Set("Content-Type", "application/json") 41 | if err := json.NewEncoder(w).Encode(&ret); err != nil { 42 | s.LogError(r, err) 43 | } 44 | } 45 | 46 | func (s *Server) createEndpoint(w http.ResponseWriter, r *http.Request) { 47 | var endpoint cosmos.Endpoint 48 | if err := json.NewDecoder(r.Body).Decode(&endpoint); err != nil { 49 | s.ReplyWithSanitizedError(w, r, cosmos.Errorf(cosmos.EINVALID, "Invalid JSON body")) 50 | return 51 | } 52 | 53 | err := s.App.CreateEndpoint(r.Context(), &endpoint) 54 | if err != nil { 55 | s.ReplyWithSanitizedError(w, r, err) 56 | return 57 | } 58 | 59 | w.Header().Set("Content-Type", "application/json") 60 | w.WriteHeader(http.StatusCreated) 61 | if err := json.NewEncoder(w).Encode(&endpoint); err != nil { 62 | s.LogError(r, err) 63 | } 64 | } 65 | 66 | func (s *Server) updateEndpoint(w http.ResponseWriter, r *http.Request) { 67 | id, err := strconv.Atoi(mux.Vars(r)["id"]) 68 | if err != nil { 69 | s.ReplyWithSanitizedError(w, r, cosmos.Errorf(cosmos.EINVALID, "Invalid endpoint ID")) 70 | return 71 | } 72 | 73 | upd := &cosmos.EndpointUpdate{} 74 | if err := json.NewDecoder(r.Body).Decode(upd); err != nil { 75 | s.ReplyWithSanitizedError(w, r, cosmos.Errorf(cosmos.EINVALID, "Invalid JSON body")) 76 | return 77 | } 78 | 79 | endpoint, err := s.App.UpdateEndpoint(r.Context(), id, upd) 80 | if err != nil { 81 | s.ReplyWithSanitizedError(w, r, err) 82 | return 83 | } 84 | 85 | w.Header().Set("Content-Type", "application/json") 86 | if err := json.NewEncoder(w).Encode(endpoint); err != nil { 87 | s.LogError(r, err) 88 | } 89 | } 90 | 91 | func (s *Server) deleteEndpoint(w http.ResponseWriter, r *http.Request) { 92 | // Parse endpoint id from path params. 93 | id, err := strconv.Atoi(mux.Vars(r)["id"]) 94 | if err != nil { 95 | s.ReplyWithSanitizedError(w, r, cosmos.Errorf(cosmos.EINVALID, "Invalid endpoint ID")) 96 | return 97 | } 98 | 99 | if err := s.App.DeleteEndpoint(r.Context(), id); err != nil { 100 | s.ReplyWithSanitizedError(w, r, err) 101 | } 102 | 103 | w.Header().Set("Content-Type", "application/json") 104 | w.Write([]byte(`{}`)) 105 | } 106 | 107 | func (s *Server) editEndpointForm(w http.ResponseWriter, r *http.Request) { 108 | id, err := strconv.Atoi(mux.Vars(r)["id"]) 109 | if err != nil { 110 | s.ReplyWithSanitizedError(w, r, cosmos.Errorf(cosmos.EINVALID, "Invalid endpoint ID")) 111 | return 112 | } 113 | 114 | endpoint, err := s.App.FindEndpointByID(r.Context(), id) 115 | if err != nil { 116 | s.ReplyWithSanitizedError(w, r, err) 117 | return 118 | } 119 | 120 | baseForm := s.App.MessageToForm(r.Context(), &endpoint.Connector.Spec, nil) 121 | baseForm.Merge(&endpoint.Config) 122 | 123 | w.Header().Set("Content-Type", "application/json") 124 | if err := json.NewEncoder(w).Encode(baseForm); err != nil { 125 | s.LogError(r, err) 126 | } 127 | } 128 | 129 | func (s *Server) rediscoverEndpoint(w http.ResponseWriter, r *http.Request) { 130 | id, err := strconv.Atoi(mux.Vars(r)["id"]) 131 | if err != nil { 132 | s.ReplyWithSanitizedError(w, r, cosmos.Errorf(cosmos.EINVALID, "Invalid endpoint ID")) 133 | return 134 | } 135 | 136 | if err := s.App.RediscoverEndpoint(r.Context(), id); err != nil { 137 | s.ReplyWithSanitizedError(w, r, err) 138 | return 139 | } 140 | } 141 | 142 | func (s *Server) catalogForm(w http.ResponseWriter, r *http.Request) { 143 | srcID, err := strconv.Atoi(mux.Vars(r)["srcID"]) 144 | if err != nil { 145 | s.ReplyWithSanitizedError(w, r, cosmos.Errorf(cosmos.EINVALID, "Invalid source endpoint ID")) 146 | return 147 | } 148 | dstID, err := strconv.Atoi(mux.Vars(r)["dstID"]) 149 | if err != nil { 150 | s.ReplyWithSanitizedError(w, r, cosmos.Errorf(cosmos.EINVALID, "Invalid destination endpoint ID")) 151 | return 152 | } 153 | 154 | srcEndpoint, err := s.App.FindEndpointByID(r.Context(), srcID) 155 | if err != nil { 156 | s.ReplyWithSanitizedError(w, r, err) 157 | return 158 | } 159 | dstEndpoint, err := s.App.FindEndpointByID(r.Context(), dstID) 160 | if err != nil { 161 | s.ReplyWithSanitizedError(w, r, err) 162 | return 163 | } 164 | 165 | createForm := s.App.MessageToForm( 166 | r.Context(), 167 | &srcEndpoint.Catalog, 168 | dstEndpoint.Connector.Spec.Spec.SupportedDestinationSyncModes, 169 | ) 170 | 171 | w.Header().Set("Content-Type", "application/json") 172 | if err := json.NewEncoder(w).Encode(createForm); err != nil { 173 | s.LogError(r, err) 174 | } 175 | } 176 | -------------------------------------------------------------------------------- /cosmos-backend/postgres/connector.go: -------------------------------------------------------------------------------- 1 | package postgres 2 | 3 | import ( 4 | "context" 5 | "cosmos" 6 | "fmt" 7 | "strings" 8 | 9 | "github.com/jackc/pgx/v4" 10 | ) 11 | 12 | func (s *DBService) FindConnectorByID(ctx context.Context, id int) (*cosmos.Connector, error) { 13 | tx, err := s.db.BeginTx(ctx, pgx.TxOptions{}) 14 | if err != nil { 15 | return nil, err 16 | } 17 | defer tx.Rollback(ctx) 18 | return findConnectorByID(ctx, tx, id) 19 | } 20 | 21 | func (s *DBService) FindConnectors(ctx context.Context, filter cosmos.ConnectorFilter) ([]*cosmos.Connector, int, error) { 22 | tx, err := s.db.BeginTx(ctx, pgx.TxOptions{}) 23 | if err != nil { 24 | return nil, 0, err 25 | } 26 | defer tx.Rollback(ctx) 27 | return findConnectors(ctx, tx, filter) 28 | } 29 | 30 | func (s *DBService) CreateConnector(ctx context.Context, connector *cosmos.Connector) error { 31 | tx, err := s.db.BeginTx(ctx, pgx.TxOptions{}) 32 | if err != nil { 33 | return err 34 | } 35 | defer tx.Rollback(ctx) 36 | 37 | if err := createConnector(ctx, tx, connector); err != nil { 38 | return err 39 | } 40 | 41 | return tx.Commit(ctx) 42 | } 43 | 44 | func (s *DBService) UpdateConnector(ctx context.Context, id int, connector *cosmos.Connector) error { 45 | tx, err := s.db.BeginTx(ctx, pgx.TxOptions{}) 46 | if err != nil { 47 | return err 48 | } 49 | defer tx.Rollback(ctx) 50 | 51 | if err := updateConnector(ctx, tx, id, connector); err != nil { 52 | return err 53 | } 54 | 55 | return tx.Commit(ctx) 56 | } 57 | 58 | func (s *DBService) DeleteConnector(ctx context.Context, id int) error { 59 | tx, err := s.db.BeginTx(ctx, pgx.TxOptions{}) 60 | if err != nil { 61 | return err 62 | } 63 | defer tx.Rollback(ctx) 64 | 65 | if err := deleteConnector(ctx, tx, id); err != nil { 66 | return err 67 | } 68 | 69 | return tx.Commit(ctx) 70 | } 71 | 72 | func findConnectorByID(ctx context.Context, tx *Tx, id int) (*cosmos.Connector, error) { 73 | connectors, totalConnectors, err := findConnectors(ctx, tx, cosmos.ConnectorFilter{ID: &id}) 74 | if err != nil { 75 | return nil, err 76 | } else if totalConnectors == 0 { 77 | return nil, cosmos.Errorf(cosmos.ENOTFOUND, "Connector not found") 78 | } 79 | return connectors[0], nil 80 | } 81 | 82 | func findConnectors(ctx context.Context, tx *Tx, filter cosmos.ConnectorFilter) ([]*cosmos.Connector, int, error) { 83 | // Build the WHERE clause. 84 | where, args, i := []string{"1 = 1"}, []interface{}{}, 1 85 | if v := filter.ID; v != nil { 86 | where, args = append(where, fmt.Sprintf("id = $%d", i)), append(args, *v) 87 | i++ 88 | } 89 | if v := filter.Name; v != nil { 90 | where, args = append(where, fmt.Sprintf("name = $%d", i)), append(args, *v) 91 | i++ 92 | } 93 | if v := filter.Type; v != nil { 94 | where, args = append(where, fmt.Sprintf("type = $%d", i)), append(args, *v) 95 | i++ 96 | } 97 | 98 | rows, err := tx.Query(ctx, ` 99 | SELECT 100 | id, 101 | name, 102 | type, 103 | docker_image_name, 104 | docker_image_tag, 105 | destination_type, 106 | spec, 107 | created_at, 108 | updated_at, 109 | COUNT(*) OVER() 110 | FROM connectors 111 | WHERE `+strings.Join(where, " AND ")+` 112 | ORDER BY name ASC 113 | `+FormatLimitOffset(filter.Limit, filter.Offset), 114 | args..., 115 | ) 116 | if err != nil { 117 | return nil, 0, err 118 | } 119 | defer rows.Close() 120 | 121 | // Iterate over the returned rows and deserialize into cosmos.Connector objects. 122 | connectors := []*cosmos.Connector{} 123 | totalConnectors := 0 124 | for rows.Next() { 125 | var connector cosmos.Connector 126 | if err := rows.Scan( 127 | &connector.ID, 128 | &connector.Name, 129 | &connector.Type, 130 | &connector.DockerImageName, 131 | &connector.DockerImageTag, 132 | &connector.DestinationType, 133 | (*Message)(&connector.Spec), 134 | (*NullTime)(&connector.CreatedAt), 135 | (*NullTime)(&connector.UpdatedAt), 136 | &totalConnectors, 137 | ); err != nil { 138 | return nil, 0, err 139 | } 140 | connectors = append(connectors, &connector) 141 | } 142 | if err := rows.Err(); err != nil { 143 | return nil, 0, err 144 | } 145 | 146 | return connectors, totalConnectors, nil 147 | } 148 | 149 | func createConnector(ctx context.Context, tx *Tx, connector *cosmos.Connector) error { 150 | // Set timestamps to current time. 151 | connector.CreatedAt = tx.now 152 | connector.UpdatedAt = connector.CreatedAt 153 | 154 | // Insert connector into database. 155 | err := tx.QueryRow(ctx, ` 156 | INSERT INTO connectors ( 157 | name, 158 | type, 159 | docker_image_name, 160 | docker_image_tag, 161 | destination_type, 162 | spec, 163 | created_at, 164 | updated_at 165 | ) 166 | VALUES ($1, $2, $3, $4, $5, $6, $7, $8) 167 | RETURNING id 168 | `, 169 | connector.Name, 170 | connector.Type, 171 | connector.DockerImageName, 172 | connector.DockerImageTag, 173 | connector.DestinationType, 174 | (*Message)(&connector.Spec), 175 | (*NullTime)(&connector.CreatedAt), 176 | (*NullTime)(&connector.UpdatedAt), 177 | ).Scan(&connector.ID) 178 | 179 | if err != nil { 180 | return FormatError(err) 181 | } 182 | 183 | return nil 184 | } 185 | 186 | func updateConnector(ctx context.Context, tx *Tx, id int, connector *cosmos.Connector) error { 187 | connector.UpdatedAt = tx.now 188 | 189 | // Execute update query. 190 | if _, err := tx.Exec(ctx, ` 191 | UPDATE connectors 192 | SET 193 | name = $1, 194 | type = $2, 195 | docker_image_name = $3, 196 | docker_image_tag = $4, 197 | destination_type = $5, 198 | spec = $6, 199 | updated_at = $7 200 | WHERE 201 | id = $8 202 | `, 203 | connector.Name, 204 | connector.Type, 205 | connector.DockerImageName, 206 | connector.DockerImageTag, 207 | connector.DestinationType, 208 | (*Message)(&connector.Spec), 209 | (*NullTime)(&connector.UpdatedAt), 210 | id, 211 | ); err != nil { 212 | return FormatError(err) 213 | } 214 | 215 | return nil 216 | } 217 | 218 | func deleteConnector(ctx context.Context, tx *Tx, id int) error { 219 | // Verify that the connector object exists. 220 | if _, err := findConnectorByID(ctx, tx, id); err != nil { 221 | return err 222 | } 223 | 224 | // Remove connector from database. 225 | if _, err := tx.Exec(ctx, ` 226 | DELETE FROM connectors 227 | WHERE id = $1 228 | `, 229 | id, 230 | ); err != nil { 231 | return err 232 | } 233 | 234 | return nil 235 | } 236 | -------------------------------------------------------------------------------- /cosmos-backend/postgres/run.go: -------------------------------------------------------------------------------- 1 | package postgres 2 | 3 | import ( 4 | "context" 5 | "cosmos" 6 | "fmt" 7 | "strings" 8 | 9 | "github.com/jackc/pgx/v4" 10 | ) 11 | 12 | func (s *DBService) FindRunByID(ctx context.Context, id int) (*cosmos.Run, error) { 13 | tx, err := s.db.BeginTx(ctx, pgx.TxOptions{}) 14 | if err != nil { 15 | return nil, err 16 | } 17 | defer tx.Rollback(ctx) 18 | return findRunByID(ctx, tx, id) 19 | } 20 | 21 | func (s *DBService) FindRuns(ctx context.Context, filter cosmos.RunFilter) ([]*cosmos.Run, int, error) { 22 | tx, err := s.db.BeginTx(ctx, pgx.TxOptions{}) 23 | if err != nil { 24 | return nil, 0, err 25 | } 26 | defer tx.Rollback(ctx) 27 | return findRuns(ctx, tx, filter, true) 28 | } 29 | 30 | func (s *DBService) CreateRun(ctx context.Context, run *cosmos.Run) error { 31 | tx, err := s.db.BeginTx(ctx, pgx.TxOptions{}) 32 | if err != nil { 33 | return err 34 | } 35 | defer tx.Rollback(ctx) 36 | 37 | if err := createRun(ctx, tx, run); err != nil { 38 | return err 39 | } 40 | 41 | return tx.Commit(ctx) 42 | } 43 | 44 | func (s *DBService) UpdateRun(ctx context.Context, id int, run *cosmos.Run) error { 45 | tx, err := s.db.BeginTx(ctx, pgx.TxOptions{}) 46 | if err != nil { 47 | return err 48 | } 49 | defer tx.Rollback(ctx) 50 | 51 | if err := updateRun(ctx, tx, id, run); err != nil { 52 | return err 53 | } 54 | 55 | return tx.Commit(ctx) 56 | } 57 | 58 | func (s *DBService) GetLastRunForSyncID(ctx context.Context, syncID int) (*cosmos.Run, error) { 59 | tx, err := s.db.BeginTx(ctx, pgx.TxOptions{}) 60 | if err != nil { 61 | return nil, err 62 | } 63 | defer tx.Rollback(ctx) 64 | return getLastRunForSyncID(ctx, tx, syncID, nil, true) 65 | } 66 | 67 | func findRunByID(ctx context.Context, tx *Tx, id int) (*cosmos.Run, error) { 68 | runs, totalRuns, err := findRuns(ctx, tx, cosmos.RunFilter{ID: &id}, true) 69 | if err != nil { 70 | return nil, err 71 | } else if totalRuns == 0 { 72 | return nil, cosmos.Errorf(cosmos.ENOTFOUND, "Run not found") 73 | } 74 | return runs[0], nil 75 | } 76 | 77 | func findRuns(ctx context.Context, tx *Tx, filter cosmos.RunFilter, wantSync bool) ([]*cosmos.Run, int, error) { 78 | // Build the WHERE clause. 79 | where, args, i := []string{"1 = 1"}, []interface{}{}, 1 80 | if v := filter.ID; v != nil { 81 | where, args = append(where, fmt.Sprintf("id = $%d", i)), append(args, *v) 82 | i++ 83 | } 84 | if v := filter.SyncID; v != nil { 85 | where, args = append(where, fmt.Sprintf("sync_id = $%d", i)), append(args, *v) 86 | i++ 87 | } 88 | if v := filter.Status; v != nil { 89 | tmp := []string{} 90 | for _, s := range v { 91 | tmp = append(tmp, fmt.Sprintf("$%d", i)) 92 | args = append(args, s) 93 | i++ 94 | } 95 | where = append(where, fmt.Sprintf("status IN (%s)", strings.Join(tmp, ", "))) 96 | } 97 | if v := filter.DateRange; v != nil { 98 | where, args = append(where, fmt.Sprintf("execution_date >= $%d", i)), append(args, v[0]) 99 | i++ 100 | where, args = append(where, fmt.Sprintf("execution_date <= $%d", i)), append(args, v[1]+"T23:59:59Z") 101 | i++ 102 | } 103 | 104 | rows, err := tx.Query(ctx, ` 105 | SELECT 106 | id, 107 | sync_id, 108 | execution_date, 109 | status, 110 | stats, 111 | options, 112 | temporal_workflow_id, 113 | temporal_run_id, 114 | COUNT(*) OVER() 115 | FROM runs 116 | WHERE `+strings.Join(where, " AND ")+` 117 | ORDER BY execution_date DESC 118 | `+FormatLimitOffset(filter.Limit, filter.Offset), 119 | args..., 120 | ) 121 | if err != nil { 122 | return nil, 0, err 123 | } 124 | defer rows.Close() 125 | 126 | // Iterate over the returned rows and deserialize into cosmos.Run objects. 127 | runs := []*cosmos.Run{} 128 | totalRuns := 0 129 | for rows.Next() { 130 | var run cosmos.Run 131 | 132 | if err := rows.Scan( 133 | &run.ID, 134 | &run.SyncID, 135 | (*NullTime)(&run.ExecutionDate), 136 | &run.Status, 137 | (*RunStats)(&run.Stats), 138 | (*RunOptions)(&run.Options), 139 | &run.TemporalWorkflowID, 140 | &run.TemporalRunID, 141 | &totalRuns, 142 | ); err != nil { 143 | return nil, 0, err 144 | } 145 | 146 | runs = append(runs, &run) 147 | } 148 | if err := rows.Err(); err != nil { 149 | return nil, 0, err 150 | } 151 | 152 | if wantSync { 153 | for _, run := range runs { 154 | // associate each run with its sync. 155 | if err := attachSync(ctx, tx, run); err != nil { 156 | return nil, 0, err 157 | } 158 | } 159 | } 160 | 161 | return runs, totalRuns, nil 162 | } 163 | 164 | func createRun(ctx context.Context, tx *Tx, run *cosmos.Run) error { 165 | run.Status = cosmos.RunStatusQueued 166 | run.TemporalWorkflowID = "" 167 | run.TemporalRunID = "" 168 | 169 | err := tx.QueryRow(ctx, ` 170 | INSERT INTO runs ( 171 | sync_id, 172 | execution_date, 173 | status, 174 | stats, 175 | options, 176 | temporal_workflow_id, 177 | temporal_run_id 178 | ) 179 | VALUES ($1, $2, $3, $4, $5, $6, $7) 180 | RETURNING id 181 | `, 182 | run.SyncID, 183 | (*NullTime)(&run.ExecutionDate), 184 | run.Status, 185 | (*RunStats)(&run.Stats), 186 | (*RunOptions)(&run.Options), 187 | run.TemporalWorkflowID, 188 | run.TemporalRunID, 189 | ).Scan(&run.ID) 190 | 191 | if err != nil { 192 | return FormatError(err) 193 | } 194 | 195 | return nil 196 | } 197 | 198 | func updateRun(ctx context.Context, tx *Tx, id int, run *cosmos.Run) error { 199 | if _, err := tx.Exec(ctx, ` 200 | UPDATE runs 201 | SET 202 | sync_id = $1, 203 | execution_date = $2, 204 | status = $3, 205 | stats = $4, 206 | options = $5, 207 | temporal_workflow_id = $6, 208 | temporal_run_id = $7 209 | WHERE 210 | id = $8 211 | `, 212 | run.SyncID, 213 | (*NullTime)(&run.ExecutionDate), 214 | run.Status, 215 | (*RunStats)(&run.Stats), 216 | (*RunOptions)(&run.Options), 217 | run.TemporalWorkflowID, 218 | run.TemporalRunID, 219 | id, 220 | ); err != nil { 221 | return FormatError(err) 222 | } 223 | 224 | return nil 225 | } 226 | 227 | func getLastRunForSyncID(ctx context.Context, tx *Tx, syncID int, status []string, wantSync bool) (*cosmos.Run, error) { 228 | runs, totalRuns, err := findRuns(ctx, tx, cosmos.RunFilter{SyncID: &syncID, Status: status, Limit: 1}, wantSync) 229 | if err != nil { 230 | return nil, err 231 | } else if totalRuns == 0 { 232 | return nil, cosmos.ErrNoPrevRun 233 | } 234 | return runs[0], nil 235 | } 236 | 237 | func attachSync(ctx context.Context, tx *Tx, run *cosmos.Run) error { 238 | sync, err := findSyncByID(ctx, tx, run.SyncID) 239 | if err != nil { 240 | return err 241 | } 242 | run.Sync = sync 243 | return nil 244 | } 245 | -------------------------------------------------------------------------------- /cosmos-frontend/src/views/EndpointsByType.vue: -------------------------------------------------------------------------------- 1 | 69 | 70 | 184 | -------------------------------------------------------------------------------- /cosmos-backend/http/server.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "context" 5 | "cosmos" 6 | "log" 7 | "net" 8 | "net/http" 9 | "os" 10 | "path/filepath" 11 | "runtime/debug" 12 | "time" 13 | 14 | "github.com/gorilla/handlers" 15 | "github.com/gorilla/mux" 16 | jsoniter "github.com/json-iterator/go" 17 | ) 18 | 19 | var json = jsoniter.ConfigDefault 20 | 21 | // Server represents a HTTP server. 22 | type Server struct { 23 | listener net.Listener 24 | server *http.Server 25 | router *mux.Router 26 | 27 | Addr string 28 | 29 | *cosmos.App 30 | } 31 | 32 | // spaHandler implements the http.Handler interface, so we can use it 33 | // to respond to HTTP requests. The path to the static directory and 34 | // path to the index file within that static directory are used to 35 | // serve the SPA in the given static directory. 36 | type spaHandler struct { 37 | staticPath string 38 | indexPath string 39 | } 40 | 41 | // ServeHTTP inspects the URL path to locate a file within the static dir 42 | // on the SPA handler. If a file is found, it will be served. If not, the 43 | // file located at the index path on the SPA handler will be served. This 44 | // is suitable behavior for serving an SPA (single page application). 45 | func (h spaHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 46 | // Get the absolute path to prevent directory traversal. 47 | path, err := filepath.Abs(r.URL.Path) 48 | if err != nil { 49 | // If we failed to get the absolute path respond with a 400 bad request and stop. 50 | http.Error(w, err.Error(), http.StatusBadRequest) 51 | return 52 | } 53 | 54 | // Prepend the path with the path to the static directory. 55 | path = filepath.Join(h.staticPath, path) 56 | 57 | // Check whether a file exists at the given path. 58 | _, err = os.Stat(path) 59 | if os.IsNotExist(err) { 60 | // File does not exist, serve index.html. 61 | http.ServeFile(w, r, filepath.Join(h.staticPath, h.indexPath)) 62 | return 63 | } else if err != nil { 64 | // If we got an error (that wasn't that the file doesn't exist) stating the 65 | // file, return a 500 internal server error and stop. 66 | http.Error(w, err.Error(), http.StatusInternalServerError) 67 | return 68 | } 69 | 70 | // Otherwise, use http.FileServer to serve the static dir. 71 | http.FileServer(http.Dir(h.staticPath)).ServeHTTP(w, r) 72 | } 73 | 74 | // NewServer returns a new instance of Server. 75 | func NewServer(addr string) *Server { 76 | s := &Server{ 77 | server: &http.Server{}, 78 | router: mux.NewRouter(), 79 | Addr: addr, 80 | } 81 | 82 | // Delegate HTTP handling to the Gorilla router. 83 | // Allow CORS (See https://www.thepolyglotdeveloper.com/2017/10/handling-cors-golang-web-application/). 84 | s.server.Handler = handlers.CORS( 85 | handlers.AllowedHeaders([]string{"X-Requested-With", "Content-Type", "Authorization"}), 86 | handlers.AllowedMethods([]string{"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"}), 87 | handlers.AllowedOrigins([]string{"*"}), 88 | )(s.router) 89 | 90 | // Register routes. 91 | r := s.router.PathPrefix("/api/v1").Subrouter() 92 | r.Use(recoveryMiddleware) 93 | s.registerConnectorRoutes(r) 94 | s.registerEndpointRoutes(r) 95 | s.registerSyncRoutes(r) 96 | s.registerRunRoutes(r) 97 | s.registerArtifactRoutes(r) 98 | 99 | // Serve SPA (Single Page Application). 100 | // See https://github.com/gorilla/mux#serving-single-page-applications 101 | spa := spaHandler{staticPath: "dist", indexPath: "index.html"} 102 | s.router.PathPrefix("/").Handler(spa) 103 | 104 | return s 105 | } 106 | 107 | // recoverMiddleware recovers from panics in HTTP handlers. 108 | func recoveryMiddleware(next http.Handler) http.Handler { 109 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 110 | defer func() { 111 | if err := recover(); err != nil { 112 | log.Printf("[http] error: %s %s: %s", r.Method, r.URL.Path, err) 113 | debug.PrintStack() 114 | w.WriteHeader(http.StatusInternalServerError) 115 | } 116 | }() 117 | 118 | next.ServeHTTP(w, r) 119 | }) 120 | } 121 | 122 | // Open begins listening on the bind address. 123 | func (s *Server) Open() (err error) { 124 | if s.listener, err = net.Listen("tcp", s.Addr); err != nil { 125 | return err 126 | } 127 | 128 | // Begin serving requests on a listener. We use Serve() instead of 129 | // ListenAndServe() because it allows us to check for listener errors 130 | // (such as trying to use an already open port) synchronously. 131 | go s.server.Serve(s.listener) 132 | 133 | return nil 134 | } 135 | 136 | // Close gracefully shuts down the HTTP server. 137 | func (s *Server) Close() error { 138 | if s.listener != nil { 139 | // Allow 30 seconds for the server to shutdown cleanly. 140 | ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) 141 | defer cancel() 142 | return s.server.Shutdown(ctx) 143 | } 144 | return nil 145 | } 146 | 147 | // LogError logs an internal error. 148 | func (s *Server) LogError(r *http.Request, err error) { 149 | log.Printf("[http] error: %s %s: %s", r.Method, r.URL.Path, err) 150 | } 151 | 152 | // ReplyWithSanitizedError sends a failure response making sure to hide sensitive internal errors 153 | // from the end user. All errors returned by the application code are, by default, internal errors. 154 | // If the application code wants to send a non-internal-error to the end-user, it must explicitly 155 | // return a cosmos.Error with the appropriate code. 156 | func (s *Server) ReplyWithSanitizedError(w http.ResponseWriter, r *http.Request, err error) { 157 | code, message := cosmos.ErrorCode(err), cosmos.ErrorMessage(err) 158 | 159 | // Log the error for application developers to examine. 160 | if code == cosmos.EINTERNAL { 161 | s.LogError(r, err) 162 | } 163 | 164 | // Send sanitized errors in the response. For internal errors, this means "Internal error". 165 | w.Header().Set("Content-Type", "application/json") 166 | w.WriteHeader(ErrorStatusCode(code)) 167 | if err := json.NewEncoder(w).Encode(&ErrorResponse{Error: message}); err != nil { 168 | s.LogError(r, err) 169 | } 170 | } 171 | 172 | // ErrorResponse represents the sanitized error sent in a response. 173 | type ErrorResponse struct { 174 | Error string `json:"error"` 175 | } 176 | 177 | // ErrorStatusCode returns the HTTP status code that matches the application-specific error code. 178 | func ErrorStatusCode(code string) int { 179 | codes := map[string]int{ 180 | cosmos.EINVALID: http.StatusBadRequest, 181 | cosmos.EINTERNAL: http.StatusInternalServerError, 182 | cosmos.ECONFLICT: http.StatusConflict, 183 | cosmos.ENOTFOUND: http.StatusNotFound, 184 | cosmos.ENOTIMPLEMENTED: http.StatusNotImplemented, 185 | } 186 | 187 | if statusCode, ok := codes[code]; ok { 188 | return statusCode 189 | } 190 | 191 | return http.StatusInternalServerError 192 | } 193 | -------------------------------------------------------------------------------- /cosmos-backend/postgres/endpoint.go: -------------------------------------------------------------------------------- 1 | package postgres 2 | 3 | import ( 4 | "context" 5 | "cosmos" 6 | "fmt" 7 | "strings" 8 | 9 | "github.com/jackc/pgx/v4" 10 | ) 11 | 12 | func (s *DBService) FindEndpointByID(ctx context.Context, id int) (*cosmos.Endpoint, error) { 13 | tx, err := s.db.BeginTx(ctx, pgx.TxOptions{}) 14 | if err != nil { 15 | return nil, err 16 | } 17 | defer tx.Rollback(ctx) 18 | return findEndpointByID(ctx, tx, id) 19 | 20 | } 21 | 22 | func (s *DBService) FindEndpoints(ctx context.Context, filter cosmos.EndpointFilter) ([]*cosmos.Endpoint, int, error) { 23 | tx, err := s.db.BeginTx(ctx, pgx.TxOptions{}) 24 | if err != nil { 25 | return nil, 0, err 26 | } 27 | defer tx.Rollback(ctx) 28 | return findEndpoints(ctx, tx, filter) 29 | } 30 | 31 | func (s *DBService) CreateEndpoint(ctx context.Context, endpoint *cosmos.Endpoint) error { 32 | tx, err := s.db.BeginTx(ctx, pgx.TxOptions{}) 33 | if err != nil { 34 | return err 35 | } 36 | defer tx.Rollback(ctx) 37 | 38 | if err := createEndpoint(ctx, tx, endpoint); err != nil { 39 | return err 40 | } 41 | 42 | return tx.Commit(ctx) 43 | } 44 | 45 | func (s *DBService) UpdateEndpoint(ctx context.Context, id int, endpoint *cosmos.Endpoint) error { 46 | tx, err := s.db.BeginTx(ctx, pgx.TxOptions{}) 47 | if err != nil { 48 | return err 49 | } 50 | defer tx.Rollback(ctx) 51 | 52 | if err := updateEndpoint(ctx, tx, id, endpoint); err != nil { 53 | return err 54 | } 55 | 56 | return tx.Commit(ctx) 57 | } 58 | 59 | func (s *DBService) DeleteEndpoint(ctx context.Context, id int) error { 60 | tx, err := s.db.BeginTx(ctx, pgx.TxOptions{}) 61 | if err != nil { 62 | return err 63 | } 64 | defer tx.Rollback(ctx) 65 | 66 | if err := deleteEndpoint(ctx, tx, id); err != nil { 67 | return err 68 | } 69 | 70 | return tx.Commit(ctx) 71 | } 72 | 73 | func findEndpointByID(ctx context.Context, tx *Tx, id int) (*cosmos.Endpoint, error) { 74 | endpoints, totalEndpoints, err := findEndpoints(ctx, tx, cosmos.EndpointFilter{ID: &id}) 75 | if err != nil { 76 | return nil, err 77 | } else if totalEndpoints == 0 { 78 | return nil, cosmos.Errorf(cosmos.ENOTFOUND, "Endpoint not found") 79 | } 80 | return endpoints[0], nil 81 | } 82 | 83 | func findEndpoints(ctx context.Context, tx *Tx, filter cosmos.EndpointFilter) ([]*cosmos.Endpoint, int, error) { 84 | // Build the WHERE clause. 85 | where, args, i := []string{"1 = 1"}, []interface{}{}, 1 86 | if v := filter.ID; v != nil { 87 | where, args = append(where, fmt.Sprintf("id = $%d", i)), append(args, *v) 88 | i++ 89 | } 90 | if v := filter.Name; v != nil { 91 | where, args = append(where, fmt.Sprintf("name = $%d", i)), append(args, *v) 92 | i++ 93 | } 94 | if v := filter.Type; v != nil { 95 | where, args = append(where, fmt.Sprintf("type = $%d", i)), append(args, *v) 96 | i++ 97 | } 98 | 99 | rows, err := tx.Query(ctx, ` 100 | SELECT 101 | id, 102 | name, 103 | type, 104 | connector_id, 105 | config, 106 | catalog, 107 | last_discovered, 108 | created_at, 109 | updated_at, 110 | COUNT(*) OVER() 111 | FROM endpoints 112 | WHERE `+strings.Join(where, " AND ")+` 113 | ORDER BY name ASC 114 | `+FormatLimitOffset(filter.Limit, filter.Offset), 115 | args..., 116 | ) 117 | if err != nil { 118 | return nil, 0, err 119 | } 120 | defer rows.Close() 121 | 122 | // Iterate over the returned rows and deserialize into cosmos.Endpoint objects. 123 | endpoints := []*cosmos.Endpoint{} 124 | totalEndpoints := 0 125 | for rows.Next() { 126 | var endpoint cosmos.Endpoint 127 | 128 | if err := rows.Scan( 129 | &endpoint.ID, 130 | &endpoint.Name, 131 | &endpoint.Type, 132 | &endpoint.ConnectorID, 133 | (*Form)(&endpoint.Config), 134 | (*Message)(&endpoint.Catalog), 135 | (*NullTime)(&endpoint.LastDiscovered), 136 | (*NullTime)(&endpoint.CreatedAt), 137 | (*NullTime)(&endpoint.UpdatedAt), 138 | &totalEndpoints, 139 | ); err != nil { 140 | return nil, 0, err 141 | } 142 | 143 | endpoints = append(endpoints, &endpoint) 144 | } 145 | if err := rows.Err(); err != nil { 146 | return nil, 0, err 147 | } 148 | 149 | // associate each endpoint with its connector. 150 | for _, endpoint := range endpoints { 151 | if err := attachConnector(ctx, tx, endpoint); err != nil { 152 | return nil, 0, err 153 | } 154 | } 155 | 156 | return endpoints, totalEndpoints, nil 157 | } 158 | 159 | func createEndpoint(ctx context.Context, tx *Tx, endpoint *cosmos.Endpoint) error { 160 | // Set timestamps to current time. 161 | endpoint.CreatedAt = tx.now 162 | endpoint.UpdatedAt = tx.now 163 | endpoint.LastDiscovered = tx.now 164 | 165 | // Insert endpoint into database. 166 | err := tx.QueryRow(ctx, ` 167 | INSERT INTO endpoints ( 168 | name, 169 | type, 170 | connector_id, 171 | config, 172 | catalog, 173 | last_discovered, 174 | created_at, 175 | updated_at 176 | ) 177 | VALUES ($1, $2, $3, $4, $5, $6, $7, $8) 178 | RETURNING id 179 | `, 180 | endpoint.Name, 181 | endpoint.Type, 182 | endpoint.ConnectorID, 183 | (*Form)(&endpoint.Config), 184 | (*Message)(&endpoint.Catalog), 185 | (*NullTime)(&endpoint.LastDiscovered), 186 | (*NullTime)(&endpoint.CreatedAt), 187 | (*NullTime)(&endpoint.UpdatedAt), 188 | ).Scan(&endpoint.ID) 189 | 190 | if err != nil { 191 | return FormatError(err) 192 | } 193 | 194 | return nil 195 | } 196 | 197 | func updateEndpoint(ctx context.Context, tx *Tx, id int, endpoint *cosmos.Endpoint) error { 198 | endpoint.UpdatedAt = tx.now 199 | 200 | // Execute update query. 201 | if _, err := tx.Exec(ctx, ` 202 | UPDATE endpoints 203 | SET 204 | name = $1, 205 | type = $2, 206 | connector_id = $3, 207 | config = $4, 208 | catalog = $5, 209 | last_discovered = $6, 210 | updated_at = $7 211 | WHERE 212 | id = $8 213 | `, 214 | endpoint.Name, 215 | endpoint.Type, 216 | endpoint.ConnectorID, 217 | (*Form)(&endpoint.Config), 218 | (*Message)(&endpoint.Catalog), 219 | (*NullTime)(&endpoint.LastDiscovered), 220 | (*NullTime)(&endpoint.UpdatedAt), 221 | id, 222 | ); err != nil { 223 | return FormatError(err) 224 | } 225 | 226 | return nil 227 | } 228 | 229 | func deleteEndpoint(ctx context.Context, tx *Tx, id int) error { 230 | // Verify that the endpoint object exists. 231 | if _, err := findEndpointByID(ctx, tx, id); err != nil { 232 | return err 233 | } 234 | 235 | // Remove endpoint from database. 236 | if _, err := tx.Exec(ctx, ` 237 | DELETE FROM endpoints 238 | WHERE id = $1 239 | `, 240 | id, 241 | ); err != nil { 242 | return err 243 | } 244 | 245 | return nil 246 | } 247 | 248 | func attachConnector(ctx context.Context, tx *Tx, endpoint *cosmos.Endpoint) error { 249 | connector, err := findConnectorByID(ctx, tx, endpoint.ConnectorID) 250 | if err != nil { 251 | return err 252 | } 253 | endpoint.Connector = connector 254 | return nil 255 | } 256 | -------------------------------------------------------------------------------- /cosmos-backend/sync.go: -------------------------------------------------------------------------------- 1 | package cosmos 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strings" 7 | "time" 8 | ) 9 | 10 | const ( 11 | NamespaceDefinitionSource = "source" 12 | NamespaceDefinitionDestination = "destination" 13 | NamespaceDefinitionCustom = "custom" 14 | ) 15 | 16 | type Sync struct { 17 | ID int `json:"id"` 18 | Name string `json:"name"` 19 | SourceEndpointID int `json:"sourceEndpointID"` 20 | DestinationEndpointID int `json:"destinationEndpointID"` 21 | ScheduleInterval int `json:"scheduleInterval"` 22 | Enabled bool `json:"enabled"` 23 | BasicNormalization bool `json:"basicNormalization"` 24 | NamespaceDefinition string `json:"namespaceDefinition"` 25 | NamespaceFormat string `json:"namespaceFormat"` 26 | StreamPrefix string `json:"streamPrefix"` 27 | State map[string]interface{} `json:"state"` 28 | Config Form `json:"config"` 29 | ConfiguredCatalog Message `json:"configuredCatalog"` 30 | CreatedAt time.Time `json:"createdAt"` 31 | UpdatedAt time.Time `json:"updatedAt"` 32 | SourceEndpoint *Endpoint `json:"sourceEndpoint"` 33 | DestinationEndpoint *Endpoint `json:"destinationEndpoint"` 34 | LastRun *Run `json:"lastRun"` 35 | LastSuccessfulRun *Run `json:"lastSuccessfulRun"` 36 | } 37 | 38 | func (s *Sync) Validate() error { 39 | if s.Name == "" { 40 | return Errorf(EINVALID, "Sync name required") 41 | } else if s.SourceEndpointID == 0 { 42 | return Errorf(EINVALID, "A source endpoint must be selected") 43 | } else if s.DestinationEndpointID == 0 { 44 | return Errorf(EINVALID, "A destination endpoint must be selected") 45 | } else if s.ScheduleInterval < 0 { 46 | return Errorf(EINVALID, "Schedule interval must be greater than or equal to 0") 47 | } else if err := s.hasValidNamespaceDefinition(); err != nil { 48 | return Errorf(EINVALID, err.Error()) 49 | } 50 | return nil 51 | } 52 | 53 | func (s *Sync) hasValidNamespaceDefinition() error { 54 | switch s.NamespaceDefinition { 55 | case NamespaceDefinitionSource, NamespaceDefinitionDestination: 56 | case NamespaceDefinitionCustom: 57 | if len(strings.TrimSpace(s.NamespaceFormat)) == 0 { 58 | return fmt.Errorf("Custom namespace definition requires a non-empty namespace format") 59 | } 60 | default: 61 | return fmt.Errorf("Invalid namespace definition: %s", s.NamespaceDefinition) 62 | } 63 | return nil 64 | } 65 | 66 | func (s *Sync) NamespaceMapper(obj interface{}) { 67 | var streamName *string 68 | var namespace **string 69 | 70 | switch v := obj.(type) { 71 | case *Stream: 72 | streamName = &v.Name 73 | namespace = &v.Namespace 74 | case *Record: 75 | streamName = &v.Stream 76 | namespace = &v.Namespace 77 | default: 78 | panic("Invalid type for namespace mapping") 79 | } 80 | 81 | *streamName = s.StreamPrefix + *streamName 82 | if s.NamespaceDefinition == NamespaceDefinitionSource { 83 | // nothing to do here. 84 | } else if s.NamespaceDefinition == NamespaceDefinitionDestination { 85 | *namespace = nil 86 | } else if s.NamespaceDefinition == NamespaceDefinitionCustom { 87 | replaceWith := "" 88 | if *namespace != nil { 89 | replaceWith = **namespace 90 | } 91 | customNamespace := strings.ReplaceAll(s.NamespaceFormat, "${SOURCE_NAMESPACE}", replaceWith) 92 | *namespace = &customNamespace 93 | } 94 | } 95 | 96 | type SyncUpdate struct { 97 | Name *string `json:"name"` 98 | Config *Form `json:"config"` 99 | ScheduleInterval *int `json:"scheduleInterval"` 100 | Enabled *bool `json:"enabled"` 101 | BasicNormalization *bool `json:"basicNormalization"` 102 | NamespaceDefinition *string `json:"namespaceDefinition"` 103 | NamespaceFormat *string `json:"namespaceFormat"` 104 | StreamPrefix *string `json:"streamPrefix"` 105 | State *map[string]interface{} `json:"state"` 106 | } 107 | 108 | type SyncFilter struct { 109 | ID *int `json:"id"` 110 | Name *string `json:"name"` 111 | 112 | Offset int `json:"offset"` 113 | Limit int `json:"limit"` 114 | } 115 | 116 | type SyncService interface { 117 | FindSyncByID(ctx context.Context, id int) (*Sync, error) 118 | FindSyncs(ctx context.Context, filter SyncFilter) ([]*Sync, int, error) 119 | CreateSync(ctx context.Context, sync *Sync) error 120 | UpdateSync(ctx context.Context, id int, sync *Sync) error 121 | DeleteSync(ctx context.Context, id int) error 122 | } 123 | 124 | func (a *App) CreateSync(ctx context.Context, sync *Sync) error { 125 | // Perform basic field validation. 126 | if err := sync.Validate(); err != nil { 127 | return err 128 | } 129 | 130 | sync.Enabled = false 131 | 132 | config, err := json.Marshal(sync.Config.ToConfiguredCatalog()) 133 | if err != nil { 134 | return err 135 | } 136 | msg, err := a.CreateMessage(ctx, config) 137 | if err != nil { 138 | return err 139 | } 140 | sync.ConfiguredCatalog = *msg 141 | 142 | return a.DBService.CreateSync(ctx, sync) 143 | } 144 | 145 | func (a *App) UpdateSync(ctx context.Context, id int, upd *SyncUpdate) (*Sync, error) { 146 | // Fetch the current sync object from the database. 147 | sync, err := a.FindSyncByID(ctx, id) 148 | if err != nil { 149 | return nil, err 150 | } 151 | 152 | // Update fields if set. 153 | if v := upd.Name; v != nil { 154 | sync.Name = *v 155 | } 156 | if v := upd.ScheduleInterval; v != nil { 157 | sync.ScheduleInterval = *v 158 | } 159 | if v := upd.Enabled; v != nil { 160 | sync.Enabled = *v 161 | } 162 | if v := upd.BasicNormalization; v != nil { 163 | sync.BasicNormalization = *v 164 | } 165 | if v := upd.NamespaceDefinition; v != nil { 166 | sync.NamespaceDefinition = *v 167 | } 168 | if v := upd.NamespaceFormat; v != nil { 169 | sync.NamespaceFormat = *v 170 | } 171 | if v := upd.StreamPrefix; v != nil { 172 | sync.StreamPrefix = *v 173 | } 174 | if v := upd.Config; v != nil { 175 | sync.Config = *v 176 | } 177 | if v := upd.State; v != nil { 178 | sync.State = *v 179 | if len(sync.State) == 0 { 180 | sync.State = nil 181 | } 182 | } 183 | 184 | // Perform basic validation to make sure that the updates are correct. 185 | if err := sync.Validate(); err != nil { 186 | return nil, err 187 | } 188 | 189 | config, err := json.Marshal(sync.Config.ToConfiguredCatalog()) 190 | if err != nil { 191 | return nil, err 192 | } 193 | msg, err := a.CreateMessage(ctx, config) 194 | if err != nil { 195 | return nil, err 196 | } 197 | sync.ConfiguredCatalog = *msg 198 | 199 | if err := a.DBService.UpdateSync(ctx, id, sync); err != nil { 200 | return nil, err 201 | } 202 | 203 | return sync, nil 204 | } 205 | -------------------------------------------------------------------------------- /cosmos-backend/form.go: -------------------------------------------------------------------------------- 1 | package cosmos 2 | 3 | import ( 4 | "regexp" 5 | ) 6 | 7 | const ( 8 | FormTypeSpec = "SPEC" 9 | FormTypeCatalog = "CATALOG" 10 | ) 11 | 12 | var ( 13 | OneOfPattern = regexp.MustCompile(`^<<\d+>>$`) 14 | ) 15 | 16 | type Form struct { 17 | Type string `json:"type,omitempty"` 18 | Spec []*FormFieldSpec `json:"spec,omitempty"` 19 | Catalog []*FormFieldCatalog `json:"catalog,omitempty"` 20 | } 21 | 22 | type FormFieldSpec struct { 23 | Path []string `json:"path"` 24 | Title string `json:"title" mapstructure:"title"` 25 | Description string `json:"description" mapstructure:"description"` 26 | Default interface{} `json:"default" mapstructure:"default"` 27 | Examples []interface{} `json:"examples" mapstructure:"examples"` 28 | Type string `json:"type" mapstructure:"type"` 29 | Enum []interface{} `json:"enum" mapstructure:"enum"` 30 | Const interface{} `json:"const" mapstructure:"const"` 31 | Secret bool `json:"secret" mapstructure:"airbyte_secret"` 32 | Order int `json:"order" mapstructure:"order"` 33 | Value interface{} `json:"value"` 34 | Multiple bool `json:"multiple"` 35 | Required bool `json:"required"` 36 | DependsOnIdx *int `json:"dependsOnIdx"` 37 | DependsOnValue []interface{} `json:"dependsOnValue"` 38 | OneOfKey bool `json:"oneOfKey"` 39 | Ignore bool `json:"ignore"` 40 | } 41 | 42 | type FormFieldCatalog struct { 43 | Stream Stream `json:"stream"` 44 | StreamNamespace *string `json:"streamNamespace"` 45 | StreamName string `json:"streamName"` 46 | IsStreamSelected bool `json:"isStreamSelected"` 47 | SyncModes [][]string `json:"syncModes"` 48 | SelectedSyncMode []string `json:"selectedSyncMode"` 49 | CursorFields [][]string `json:"cursorFields"` 50 | SelectedCursorField []string `json:"selectedCursorField"` 51 | PrimaryKeys [][]string `json:"primaryKeys"` 52 | SelectedPrimaryKey [][]string `json:"selectedPrimaryKey"` 53 | } 54 | 55 | func (f *FormFieldSpec) EnumContainsValue(value interface{}) bool { 56 | for _, e := range f.Enum { 57 | if e == value { 58 | return true 59 | } 60 | } 61 | return false 62 | } 63 | 64 | func (f *FormFieldSpec) DependsOnValuesIncludes(value interface{}) bool { 65 | for _, e := range f.DependsOnValue { 66 | if e == value { 67 | return true 68 | } 69 | } 70 | return false 71 | } 72 | 73 | func (f *FormFieldCatalog) IsSyncModeAvailable(syncMode []string) bool { 74 | for _, m := range f.SyncModes { 75 | if testEq(m, syncMode) { 76 | return true 77 | } 78 | } 79 | return false 80 | } 81 | 82 | func (f *FormFieldCatalog) IsCursorFieldAvailable(cursorField []string) bool { 83 | for _, m := range f.CursorFields { 84 | if testEq(m, cursorField) { 85 | return true 86 | } 87 | } 88 | return false 89 | } 90 | 91 | func (f *FormFieldCatalog) IsPrimaryKeyAvailable(primaryKey [][]string) bool { 92 | for _, p := range primaryKey { 93 | found := false 94 | for _, m := range f.PrimaryKeys { 95 | if testEq(m, p) { 96 | found = true 97 | break 98 | } 99 | } 100 | if !found { 101 | return false 102 | } 103 | } 104 | return true 105 | } 106 | 107 | func (f *Form) ToSpec() map[string]interface{} { 108 | result := map[string]interface{}{} 109 | 110 | for _, field := range f.Spec { 111 | if field.Ignore { 112 | continue 113 | } 114 | if field.Value == nil && !field.Required { 115 | continue 116 | } 117 | if field.DependsOnIdx != nil && !field.DependsOnValuesIncludes(f.Spec[*field.DependsOnIdx].Value) { 118 | continue 119 | } 120 | 121 | m := result 122 | for _, p := range field.Path[:len(field.Path)-1] { 123 | if OneOfPattern.MatchString(p) { 124 | continue 125 | } 126 | if m[p] == nil { 127 | m[p] = map[string]interface{}{} 128 | } 129 | m = m[p].(map[string]interface{}) 130 | } 131 | m[field.Path[len(field.Path)-1]] = field.Value 132 | } 133 | 134 | return result 135 | } 136 | 137 | func (f *Form) ToConfiguredCatalog() map[string]interface{} { 138 | result := map[string]interface{}{ 139 | "type": MessageTypeConfiguredCatalog, 140 | "configuredCatalog": map[string][]interface{}{ 141 | "streams": nil, 142 | }, 143 | } 144 | 145 | cc := result["configuredCatalog"].(map[string][]interface{}) 146 | 147 | for _, field := range f.Catalog { 148 | if !field.IsStreamSelected { 149 | continue 150 | } 151 | 152 | m := map[string]interface{}{} 153 | 154 | m["stream"] = field.Stream 155 | 156 | m["sync_mode"] = field.SelectedSyncMode[0] 157 | if field.SelectedSyncMode[0] == SyncModeIncremental && 158 | len(field.SelectedCursorField) != 0 { 159 | m["cursor_field"] = field.SelectedCursorField 160 | } 161 | 162 | m["destination_sync_mode"] = field.SelectedSyncMode[1] 163 | if (field.SelectedSyncMode[1] == DestinationSyncModeAppendDedup || 164 | field.SelectedSyncMode[1] == DestinationSyncModeUpsertDedup) && 165 | len(field.SelectedPrimaryKey) != 0 { 166 | m["primary_key"] = field.SelectedPrimaryKey 167 | } 168 | 169 | cc["streams"] = append(cc["streams"], m) 170 | } 171 | 172 | return result 173 | } 174 | 175 | func (f *Form) Merge(patch *Form) { 176 | switch f.Type { 177 | case FormTypeSpec: 178 | var isMatch func(baseField, patchField *FormFieldSpec) bool 179 | isMatch = func(baseField, patchField *FormFieldSpec) bool { 180 | if !testEq(baseField.Path, patchField.Path) { 181 | return false 182 | } 183 | if baseField.Type != patchField.Type { 184 | return false 185 | } 186 | if baseField.Multiple != patchField.Multiple { 187 | return false 188 | } 189 | if len(baseField.Enum) != 0 && !baseField.EnumContainsValue(patchField.Value) { 190 | return false 191 | } 192 | if (baseField.DependsOnIdx == nil) != (patchField.DependsOnIdx == nil) { 193 | return false 194 | } 195 | if baseField.DependsOnIdx != nil && 196 | !isMatch(f.Spec[*baseField.DependsOnIdx], patch.Spec[*patchField.DependsOnIdx]) { 197 | return false 198 | } 199 | return true 200 | } 201 | 202 | for _, baseField := range f.Spec { 203 | for _, patchField := range patch.Spec { 204 | if isMatch(baseField, patchField) { 205 | baseField.Value = patchField.Value 206 | break 207 | } 208 | } 209 | } 210 | 211 | case FormTypeCatalog: 212 | isMatch := func(baseField, patchField *FormFieldCatalog) bool { 213 | if baseField.StreamName != patchField.StreamName { 214 | return false 215 | } 216 | if !baseField.IsSyncModeAvailable(patchField.SelectedSyncMode) { 217 | return false 218 | } 219 | if patchField.SelectedCursorField != nil && 220 | !baseField.IsCursorFieldAvailable(patchField.SelectedCursorField) { 221 | return false 222 | } 223 | if patchField.SelectedPrimaryKey != nil && 224 | !baseField.IsPrimaryKeyAvailable(patchField.SelectedPrimaryKey) { 225 | return false 226 | } 227 | return true 228 | } 229 | for _, baseField := range f.Catalog { 230 | for _, patchField := range patch.Catalog { 231 | if isMatch(baseField, patchField) { 232 | baseField.IsStreamSelected = patchField.IsStreamSelected 233 | baseField.SelectedSyncMode = patchField.SelectedSyncMode 234 | baseField.SelectedCursorField = patchField.SelectedCursorField 235 | baseField.SelectedPrimaryKey = patchField.SelectedPrimaryKey 236 | break 237 | } 238 | } 239 | } 240 | 241 | default: 242 | panic("Unhandled form type in merge") 243 | } 244 | } 245 | 246 | func testEq(a, b []string) bool { 247 | if (a == nil) != (b == nil) { 248 | return false 249 | } 250 | if len(a) != len(b) { 251 | return false 252 | } 253 | for i := range a { 254 | if a[i] != b[i] { 255 | return false 256 | } 257 | } 258 | return true 259 | } 260 | -------------------------------------------------------------------------------- /cosmos-backend/postgres/sync.go: -------------------------------------------------------------------------------- 1 | package postgres 2 | 3 | import ( 4 | "context" 5 | "cosmos" 6 | "errors" 7 | "fmt" 8 | "strings" 9 | 10 | "github.com/jackc/pgx/v4" 11 | ) 12 | 13 | func (s *DBService) FindSyncByID(ctx context.Context, id int) (*cosmos.Sync, error) { 14 | tx, err := s.db.BeginTx(ctx, pgx.TxOptions{}) 15 | if err != nil { 16 | return nil, err 17 | } 18 | defer tx.Rollback(ctx) 19 | return findSyncByID(ctx, tx, id) 20 | } 21 | 22 | func (s *DBService) FindSyncs(ctx context.Context, filter cosmos.SyncFilter) ([]*cosmos.Sync, int, error) { 23 | tx, err := s.db.BeginTx(ctx, pgx.TxOptions{}) 24 | if err != nil { 25 | return nil, 0, err 26 | } 27 | defer tx.Rollback(ctx) 28 | return findSyncs(ctx, tx, filter) 29 | } 30 | 31 | func (s *DBService) CreateSync(ctx context.Context, sync *cosmos.Sync) error { 32 | tx, err := s.db.BeginTx(ctx, pgx.TxOptions{}) 33 | if err != nil { 34 | return err 35 | } 36 | defer tx.Rollback(ctx) 37 | 38 | if err := createSync(ctx, tx, sync); err != nil { 39 | return err 40 | } 41 | 42 | return tx.Commit(ctx) 43 | } 44 | 45 | func (s *DBService) UpdateSync(ctx context.Context, id int, sync *cosmos.Sync) error { 46 | tx, err := s.db.BeginTx(ctx, pgx.TxOptions{}) 47 | if err != nil { 48 | return err 49 | } 50 | defer tx.Rollback(ctx) 51 | 52 | if err := updateSync(ctx, tx, id, sync); err != nil { 53 | return err 54 | } 55 | 56 | return tx.Commit(ctx) 57 | } 58 | 59 | func (s *DBService) DeleteSync(ctx context.Context, id int) error { 60 | tx, err := s.db.BeginTx(ctx, pgx.TxOptions{}) 61 | if err != nil { 62 | return err 63 | } 64 | defer tx.Rollback(ctx) 65 | 66 | if err := deleteSync(ctx, tx, id); err != nil { 67 | return err 68 | } 69 | 70 | return tx.Commit(ctx) 71 | } 72 | 73 | func findSyncByID(ctx context.Context, tx *Tx, id int) (*cosmos.Sync, error) { 74 | syncs, totalSyncs, err := findSyncs(ctx, tx, cosmos.SyncFilter{ID: &id}) 75 | if err != nil { 76 | return nil, err 77 | } else if totalSyncs == 0 { 78 | return nil, cosmos.Errorf(cosmos.ENOTFOUND, "Sync not found") 79 | } 80 | return syncs[0], nil 81 | } 82 | 83 | func findSyncs(ctx context.Context, tx *Tx, filter cosmos.SyncFilter) ([]*cosmos.Sync, int, error) { 84 | // Build the WHERE clause. 85 | where, args, i := []string{"1 = 1"}, []interface{}{}, 1 86 | if v := filter.ID; v != nil { 87 | where, args = append(where, fmt.Sprintf("id = $%d", i)), append(args, *v) 88 | i++ 89 | } 90 | if v := filter.Name; v != nil { 91 | where, args = append(where, fmt.Sprintf("name = $%d", i)), append(args, *v) 92 | i++ 93 | } 94 | 95 | rows, err := tx.Query(ctx, ` 96 | SELECT 97 | id, 98 | name, 99 | source_endpoint_id, 100 | destination_endpoint_id, 101 | schedule_interval, 102 | enabled, 103 | basic_normalization, 104 | namespace_definition, 105 | namespace_format, 106 | stream_prefix, 107 | state, 108 | config, 109 | configured_catalog, 110 | created_at, 111 | updated_at, 112 | COUNT(*) OVER() 113 | FROM syncs 114 | WHERE `+strings.Join(where, " AND ")+` 115 | ORDER BY name ASC 116 | `+FormatLimitOffset(filter.Limit, filter.Offset), 117 | args..., 118 | ) 119 | if err != nil { 120 | return nil, 0, err 121 | } 122 | defer rows.Close() 123 | 124 | // Iterate over the returned rows and deserialize into cosmos.Sync objects. 125 | syncs := []*cosmos.Sync{} 126 | totalSyncs := 0 127 | for rows.Next() { 128 | var sync cosmos.Sync 129 | 130 | if err := rows.Scan( 131 | &sync.ID, 132 | &sync.Name, 133 | &sync.SourceEndpointID, 134 | &sync.DestinationEndpointID, 135 | &sync.ScheduleInterval, 136 | &sync.Enabled, 137 | &sync.BasicNormalization, 138 | &sync.NamespaceDefinition, 139 | &sync.NamespaceFormat, 140 | &sync.StreamPrefix, 141 | (*Map)(&sync.State), 142 | (*Form)(&sync.Config), 143 | (*Message)(&sync.ConfiguredCatalog), 144 | (*NullTime)(&sync.CreatedAt), 145 | (*NullTime)(&sync.UpdatedAt), 146 | &totalSyncs, 147 | ); err != nil { 148 | return nil, 0, err 149 | } 150 | 151 | syncs = append(syncs, &sync) 152 | } 153 | if err := rows.Err(); err != nil { 154 | return nil, 0, err 155 | } 156 | 157 | for _, sync := range syncs { 158 | // associate each sync with its endpoints. 159 | if err := attachEndpoints(ctx, tx, sync); err != nil { 160 | return nil, 0, err 161 | } 162 | 163 | // associate each sync with its last run. 164 | if err := attachLastRun(ctx, tx, sync); err != nil { 165 | return nil, 0, err 166 | } 167 | 168 | // associate each sync with its last successful run. 169 | if err := attachLastSuccessfulRun(ctx, tx, sync); err != nil { 170 | return nil, 0, err 171 | } 172 | } 173 | 174 | return syncs, totalSyncs, nil 175 | } 176 | 177 | func createSync(ctx context.Context, tx *Tx, sync *cosmos.Sync) error { 178 | // Set timestamps to current time. 179 | sync.CreatedAt = tx.now 180 | sync.UpdatedAt = sync.CreatedAt 181 | 182 | // Insert sync into database. 183 | err := tx.QueryRow(ctx, ` 184 | INSERT INTO syncs ( 185 | name, 186 | source_endpoint_id, 187 | destination_endpoint_id, 188 | schedule_interval, 189 | enabled, 190 | basic_normalization, 191 | namespace_definition, 192 | namespace_format, 193 | stream_prefix, 194 | state, 195 | config, 196 | configured_catalog, 197 | created_at, 198 | updated_at 199 | ) 200 | VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) 201 | RETURNING id 202 | `, 203 | sync.Name, 204 | sync.SourceEndpointID, 205 | sync.DestinationEndpointID, 206 | sync.ScheduleInterval, 207 | sync.Enabled, 208 | sync.BasicNormalization, 209 | sync.NamespaceDefinition, 210 | sync.NamespaceFormat, 211 | sync.StreamPrefix, 212 | (*Map)(&sync.State), 213 | (*Form)(&sync.Config), 214 | (*Message)(&sync.ConfiguredCatalog), 215 | (*NullTime)(&sync.CreatedAt), 216 | (*NullTime)(&sync.UpdatedAt), 217 | ).Scan(&sync.ID) 218 | 219 | if err != nil { 220 | return FormatError(err) 221 | } 222 | 223 | return nil 224 | } 225 | 226 | func updateSync(ctx context.Context, tx *Tx, id int, sync *cosmos.Sync) error { 227 | sync.UpdatedAt = tx.now 228 | 229 | // Execute update query. 230 | if _, err := tx.Exec(ctx, ` 231 | UPDATE syncs 232 | SET 233 | name = $1, 234 | source_endpoint_id = $2, 235 | destination_endpoint_id = $3, 236 | schedule_interval = $4, 237 | enabled = $5, 238 | basic_normalization = $6, 239 | namespace_definition = $7, 240 | namespace_format = $8, 241 | stream_prefix = $9, 242 | state = $10, 243 | config = $11, 244 | configured_catalog = $12, 245 | updated_at = $13 246 | WHERE 247 | id = $14 248 | `, 249 | sync.Name, 250 | sync.SourceEndpointID, 251 | sync.DestinationEndpointID, 252 | sync.ScheduleInterval, 253 | sync.Enabled, 254 | sync.BasicNormalization, 255 | sync.NamespaceDefinition, 256 | sync.NamespaceFormat, 257 | sync.StreamPrefix, 258 | (*Map)(&sync.State), 259 | (*Form)(&sync.Config), 260 | (*Message)(&sync.ConfiguredCatalog), 261 | (*NullTime)(&sync.UpdatedAt), 262 | id, 263 | ); err != nil { 264 | return FormatError(err) 265 | } 266 | 267 | return nil 268 | } 269 | 270 | func deleteSync(ctx context.Context, tx *Tx, id int) error { 271 | // Verify that the sync object exists. 272 | if _, err := findSyncByID(ctx, tx, id); err != nil { 273 | return err 274 | } 275 | 276 | // Remove sync from database. 277 | if _, err := tx.Exec(ctx, ` 278 | DELETE FROM syncs 279 | WHERE id = $1 280 | `, 281 | id, 282 | ); err != nil { 283 | return err 284 | } 285 | 286 | return nil 287 | } 288 | 289 | func attachEndpoints(ctx context.Context, tx *Tx, sync *cosmos.Sync) error { 290 | endpoint, err := findEndpointByID(ctx, tx, sync.SourceEndpointID) 291 | if err != nil { 292 | return err 293 | } 294 | sync.SourceEndpoint = endpoint 295 | 296 | endpoint, err = findEndpointByID(ctx, tx, sync.DestinationEndpointID) 297 | if err != nil { 298 | return err 299 | } 300 | sync.DestinationEndpoint = endpoint 301 | 302 | return nil 303 | } 304 | 305 | func attachLastRun(ctx context.Context, tx *Tx, sync *cosmos.Sync) error { 306 | run, err := getLastRunForSyncID(ctx, tx, sync.ID, nil, false) 307 | if err != nil && !errors.Is(err, cosmos.ErrNoPrevRun) { 308 | return err 309 | } 310 | sync.LastRun = run 311 | return nil 312 | } 313 | 314 | func attachLastSuccessfulRun(ctx context.Context, tx *Tx, sync *cosmos.Sync) error { 315 | run, err := getLastRunForSyncID(ctx, tx, sync.ID, []string{cosmos.RunStatusSuccess}, false) 316 | if err != nil && !errors.Is(err, cosmos.ErrNoPrevRun) { 317 | return err 318 | } 319 | sync.LastSuccessfulRun = run 320 | return nil 321 | } 322 | -------------------------------------------------------------------------------- /cosmos-frontend/src/components/EditConnector.vue: -------------------------------------------------------------------------------- 1 | 75 | 76 | 219 | -------------------------------------------------------------------------------- /cosmos-frontend/src/views/Syncs.vue: -------------------------------------------------------------------------------- 1 | 88 | 89 | 232 | 233 | 256 | -------------------------------------------------------------------------------- /cosmos-backend/postgres/postgres.go: -------------------------------------------------------------------------------- 1 | package postgres 2 | 3 | import ( 4 | "context" 5 | "cosmos" 6 | "database/sql/driver" 7 | "embed" 8 | "fmt" 9 | "io/fs" 10 | "sort" 11 | "strings" 12 | "time" 13 | 14 | "github.com/jackc/pgx/v4" 15 | "github.com/jackc/pgx/v4/pgxpool" 16 | jsoniter "github.com/json-iterator/go" 17 | ) 18 | 19 | var json = jsoniter.ConfigDefault 20 | 21 | // DB represents the database connection. 22 | type DB struct { 23 | db *pgxpool.Pool 24 | url string 25 | now func() time.Time 26 | should_migrate bool 27 | } 28 | 29 | // NewDB returns a new instance of DB associated with the given URL. 30 | func NewDB(url string, should_migrate bool) *DB { 31 | return &DB{ 32 | url: url, 33 | now: time.Now, 34 | should_migrate: should_migrate, 35 | } 36 | } 37 | 38 | // Open opens the database connection. 39 | func (db *DB) Open() (err error) { 40 | if db.url == "" { 41 | return fmt.Errorf("DB url required") 42 | } 43 | 44 | // Connect to the database. 45 | if db.db, err = pgxpool.Connect(context.Background(), db.url); err != nil { 46 | return err 47 | } 48 | 49 | // Perform database migrations. 50 | if err := db.migrate(); err != nil { 51 | return fmt.Errorf("migrate: %w", err) 52 | } 53 | 54 | return nil 55 | } 56 | 57 | // Close closes the database connection. 58 | func (db *DB) Close() error { 59 | // Close database connection. 60 | if db.db != nil { 61 | db.db.Close() 62 | } 63 | return nil 64 | } 65 | 66 | //go:embed migrations/*.sql 67 | var migrationsFS embed.FS 68 | 69 | // migrate performs database migrations. 70 | // 71 | // Migration files are embedded in the postgres/migrations folder and are executed 72 | // in lexicographic order. 73 | // 74 | // Once a migration is run, its name is stored in the 'migrations' table so that 75 | // it is not re-executed. 76 | func (db *DB) migrate() error { 77 | if !db.should_migrate { 78 | return nil 79 | } 80 | 81 | // Create a publication for the supabase realtime server. There is no "IF NOT 82 | // EXISTS" clause available during create, so drop existing publication before 83 | // creating a new one. 84 | if _, err := db.db.Exec(context.Background(), `DROP PUBLICATION IF EXISTS supabase_realtime;`); err != nil { 85 | return fmt.Errorf("failed to drop publication for supabase realtime: %w", err) 86 | } 87 | if _, err := db.db.Exec(context.Background(), `CREATE PUBLICATION supabase_realtime FOR ALL TABLES;`); err != nil { 88 | return fmt.Errorf("failed to create publication for supabase realtime: %w", err) 89 | } 90 | 91 | // Create the 'migrations' table if it doesn't already exist. 92 | if _, err := db.db.Exec(context.Background(), `CREATE TABLE IF NOT EXISTS migrations (name TEXT PRIMARY KEY);`); err != nil { 93 | return fmt.Errorf("failed to create migrations table: %w", err) 94 | } 95 | 96 | // Read the migration files from the embedded filesystem. 97 | // Sort the migrations in lexicographic order. 98 | names, err := fs.Glob(migrationsFS, "migrations/*.sql") 99 | if err != nil { 100 | return err 101 | } 102 | sort.Strings(names) 103 | 104 | // Execute all migrations. 105 | if err := db.migrateFiles(names); err != nil { 106 | return fmt.Errorf("migration error: err=%w", err) 107 | } 108 | 109 | return nil 110 | } 111 | 112 | func (db *DB) migrateFiles(names []string) error { 113 | tx, err := db.db.Begin(context.Background()) 114 | if err != nil { 115 | return err 116 | } 117 | defer tx.Rollback(context.Background()) 118 | 119 | // Acquire a table-level lock which will be release when the transaction ends. 120 | if _, err := tx.Exec(context.Background(), `LOCK TABLE migrations`); err != nil { 121 | return err 122 | } 123 | 124 | for _, name := range names { 125 | // Make sure that the migration hasn't already been run. 126 | var n int 127 | if err := tx.QueryRow(context.Background(), `SELECT COUNT(*) FROM migrations WHERE name = $1`, name).Scan(&n); err != nil { 128 | return err 129 | } else if n != 0 { 130 | // Migration has already been run. Nothing more to do. 131 | return nil 132 | } 133 | 134 | // Read and execute the migration file. 135 | if buf, err := fs.ReadFile(migrationsFS, name); err != nil { 136 | return err 137 | } else if _, err := tx.Exec(context.Background(), string(buf)); err != nil { 138 | return err 139 | } 140 | 141 | // Insert a record into the migrations table to prevent re-running the migration. 142 | if _, err := tx.Exec(context.Background(), `INSERT INTO migrations (name) VALUES($1)`, name); err != nil { 143 | return err 144 | } 145 | } 146 | 147 | return tx.Commit(context.Background()) 148 | } 149 | 150 | // Tx wraps the sql Tx object to provide a transaction start timestamp. 151 | // Useful when populating CreatedAt and UpdatedAt columns. 152 | type Tx struct { 153 | pgx.Tx 154 | now time.Time 155 | } 156 | 157 | // BeginTx returns a wrapped sql Tx object containing the transaction start timestamp. 158 | func (db *DB) BeginTx(ctx context.Context, opts pgx.TxOptions) (*Tx, error) { 159 | tx, err := db.db.BeginTx(ctx, opts) 160 | if err != nil { 161 | return nil, err 162 | } 163 | 164 | return &Tx{ 165 | Tx: tx, 166 | now: db.now().UTC().Truncate(time.Second), 167 | }, nil 168 | } 169 | 170 | // NullTime represents a helper wrapper for time.Time. 171 | // It automatically converts to/from RFC 3339 format. 172 | // Also supports NULL for zero time. 173 | type NullTime time.Time 174 | 175 | // Scan reads a RFC 3339 formatted time (or NULL) from the database. 176 | func (n *NullTime) Scan(value interface{}) error { 177 | if value == nil { 178 | *n = NullTime(time.Time{}) 179 | return nil 180 | } else if s, ok := value.(string); ok { 181 | if t, err := time.Parse(time.RFC3339, s); err != nil { 182 | return fmt.Errorf("NullTime: cannot scan from string value %s", s) 183 | } else { 184 | *n = NullTime(t) 185 | return nil 186 | } 187 | } 188 | 189 | return fmt.Errorf("NullTime: cannot scan value of type %T", value) 190 | } 191 | 192 | // Value writes a RFC 3339 formatted time (or NULL) to the database. 193 | func (n *NullTime) Value() (driver.Value, error) { 194 | if n == nil || time.Time(*n).IsZero() { 195 | return nil, nil 196 | } 197 | 198 | return time.Time(*n).UTC().Format(time.RFC3339), nil 199 | } 200 | 201 | func unmarshal(value, target interface{}) error { 202 | if s, ok := value.(string); ok { 203 | if err := json.Unmarshal([]byte(s), target); err != nil { 204 | return fmt.Errorf("postgres: cannot scan from string value %s", s) 205 | } 206 | return nil 207 | } 208 | return fmt.Errorf("postgres: cannot scan value of type %T", value) 209 | } 210 | 211 | func marshal(value interface{}) (interface{}, error) { 212 | out, err := json.Marshal(value) 213 | if err != nil { 214 | return nil, fmt.Errorf("postgres: cannot marshal value") 215 | } 216 | return string(out), nil 217 | } 218 | 219 | // Message represents a helper wrapper for cosmos.Message. 220 | // It automatically converts to/from string. 221 | type Message cosmos.Message 222 | 223 | func (m *Message) Scan(value interface{}) error { 224 | return unmarshal(value, m) 225 | } 226 | 227 | func (m *Message) Value() (driver.Value, error) { 228 | return marshal(m) 229 | } 230 | 231 | // Form represents a helper wrapper for cosmos.Form. 232 | // It automatically converts to/from string. 233 | type Form cosmos.Form 234 | 235 | func (f *Form) Scan(value interface{}) error { 236 | return unmarshal(value, f) 237 | } 238 | 239 | func (f *Form) Value() (driver.Value, error) { 240 | return marshal(f) 241 | } 242 | 243 | // RunStats represents a helper wrapper for cosmos.RunStats. 244 | // It automatically converts to/from string. 245 | type RunStats cosmos.RunStats 246 | 247 | func (r *RunStats) Scan(value interface{}) error { 248 | return unmarshal(value, r) 249 | } 250 | 251 | func (r *RunStats) Value() (driver.Value, error) { 252 | return marshal(r) 253 | } 254 | 255 | // RunOptions represents a helper wrapper for cosmos.RunOptions. 256 | // It automatically converts to/from string. 257 | type RunOptions cosmos.RunOptions 258 | 259 | func (r *RunOptions) Scan(value interface{}) error { 260 | return unmarshal(value, r) 261 | } 262 | 263 | func (r *RunOptions) Value() (driver.Value, error) { 264 | return marshal(r) 265 | } 266 | 267 | // Map represents a helper wrapper for map[string]interface{}. 268 | // It automatically converts to/from string. 269 | type Map map[string]interface{} 270 | 271 | func (m *Map) Scan(value interface{}) error { 272 | return unmarshal(value, m) 273 | } 274 | 275 | func (m *Map) Value() (driver.Value, error) { 276 | return marshal(m) 277 | } 278 | 279 | // FormatLimitOffset returns a formatted string containing the LIMIT and OFFSET. 280 | func FormatLimitOffset(limit, offset int) string { 281 | if limit > 0 && offset > 0 { 282 | return fmt.Sprintf("LIMIT %d OFFSET %d", limit, offset) 283 | } else if limit > 0 { 284 | return fmt.Sprintf("LIMIT %d", limit) 285 | } else if offset > 0 { 286 | return fmt.Sprintf("OFFSET %d", offset) 287 | } 288 | return "" 289 | } 290 | 291 | // FormatError converts the error into proper application errors (cosmos.Error) where appropriate. 292 | func FormatError(err error) error { 293 | errStr := err.Error() 294 | if strings.Contains(errStr, `violates unique constraint "connectors_name_type_key"`) { 295 | return cosmos.Errorf(cosmos.ECONFLICT, "Connector already exists") 296 | } else if strings.Contains(errStr, `violates unique constraint "connectors_docker_image_name_docker_image_tag_key"`) { 297 | return cosmos.Errorf(cosmos.ECONFLICT, "Connector already exists") 298 | } else if strings.Contains(errStr, `violates unique constraint "endpoints_name_type_key"`) { 299 | return cosmos.Errorf(cosmos.ECONFLICT, "Endpoint already exists") 300 | } else if strings.Contains(errStr, `violates unique constraint "syncs_name_key"`) { 301 | return cosmos.Errorf(cosmos.ECONFLICT, "Sync already exists") 302 | } else if strings.Contains(errStr, `violates unique constraint "runs_sync_id_execution_date_key"`) { 303 | return cosmos.Errorf(cosmos.ECONFLICT, "Run already exists") 304 | } 305 | return err 306 | } 307 | -------------------------------------------------------------------------------- /cosmos-frontend/src/views/Runs.vue: -------------------------------------------------------------------------------- 1 | 106 | 107 | 284 | 285 | 308 | -------------------------------------------------------------------------------- /cosmos-frontend/src/components/CreateEndpoint.vue: -------------------------------------------------------------------------------- 1 | 145 | 146 | 147 | 268 | -------------------------------------------------------------------------------- /cosmos-backend/postgres/migrations/0000000000.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE connectors ( 2 | id SERIAL PRIMARY KEY, 3 | name TEXT NOT NULL, 4 | type TEXT NOT NULL, 5 | docker_image_name TEXT NOT NULL, 6 | docker_image_tag TEXT NOT NULL, 7 | destination_type TEXT NOT NULL, 8 | spec TEXT NOT NULL, 9 | created_at TEXT DEFAULT TO_CHAR(CURRENT_TIMESTAMP, 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), 10 | updated_at TEXT DEFAULT TO_CHAR(CURRENT_TIMESTAMP, 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), 11 | 12 | UNIQUE(name, type), 13 | UNIQUE(docker_image_name, docker_image_tag) 14 | ); 15 | 16 | CREATE INDEX connectors_type_idx ON connectors (type); 17 | 18 | CREATE TABLE endpoints ( 19 | id SERIAL PRIMARY KEY, 20 | name TEXT NOT NULL, 21 | type TEXT NOT NULL, 22 | connector_id INT NOT NULL REFERENCES connectors (id) ON DELETE CASCADE, 23 | config TEXT NOT NULL, 24 | catalog TEXT NOT NULL, 25 | last_discovered TEXT NOT NULL, 26 | created_at TEXT NOT NULL, 27 | updated_at TEXT NOT NULL, 28 | 29 | UNIQUE(name, type) 30 | ); 31 | 32 | CREATE INDEX endpoints_type_idx ON endpoints (type); 33 | 34 | CREATE TABLE syncs ( 35 | id SERIAL PRIMARY KEY, 36 | name TEXT NOT NULL, 37 | source_endpoint_id INT NOT NULL REFERENCES endpoints (id) ON DELETE CASCADE, 38 | destination_endpoint_id INT NOT NULL REFERENCES endpoints (id) ON DELETE CASCADE, 39 | schedule_interval INT NOT NULL, 40 | enabled BOOLEAN NOT NULL, 41 | basic_normalization BOOLEAN NOT NULL, 42 | namespace_definition TEXT NOT NULL, 43 | namespace_format TEXT NOT NULL, 44 | stream_prefix TEXT NOT NULL, 45 | state TEXT NOT NULL, 46 | config TEXT NOT NULL, 47 | configured_catalog TEXT NOT NULL, 48 | created_at TEXT NOT NULL, 49 | updated_at TEXT NOT NULL, 50 | 51 | UNIQUE(name) 52 | ); 53 | 54 | CREATE TABLE runs ( 55 | id SERIAL PRIMARY KEY, 56 | sync_id INT NOT NULL REFERENCES syncs (id) ON DELETE CASCADE, 57 | execution_date TEXT NOT NULL, 58 | status TEXT NOT NULL, 59 | stats TEXT NOT NULL, 60 | options TEXT NOT NULL, 61 | temporal_workflow_id TEXT NOT NULL, 62 | temporal_run_id TEXT NOT NULL, 63 | 64 | UNIQUE(sync_id, execution_date) 65 | ); 66 | 67 | CREATE INDEX runs_status_idx ON runs (status); 68 | CREATE INDEX runs_sync_id_idx ON runs (sync_id); 69 | CREATE UNIQUE INDEX runs_sync_id_execution_date_idx ON runs (sync_id, execution_date); 70 | 71 | 72 | 73 | INSERT INTO connectors(name, type, docker_image_name, docker_image_tag, destination_type, spec) VALUES 74 | ('Local JSON', 'destination', 'airbyte/destination-local-json', '0.2.8', 'other', '{"type": "SPEC"}'), 75 | ('Local CSV', 'destination', 'airbyte/destination-csv', '0.2.8', 'other', '{"type": "SPEC"}'), 76 | ('Postgres', 'destination', 'airbyte/destination-postgres', '0.3.8', 'postgres', '{"type": "SPEC"}'), 77 | ('BigQuery', 'destination', 'airbyte/destination-bigquery', '0.3.8', 'bigquery', '{"type": "SPEC"}'), 78 | ('BigQuery (denormalized typed struct)', 'destination', 'airbyte/destination-bigquery-denormalized', '0.1.1', 'other', '{"type": "SPEC"}'), 79 | ('Google Cloud Storage (GCS)', 'destination', 'airbyte/destination-gcs', '0.1.0', 'other', '{"type": "SPEC"}'), 80 | ('Google PubSub', 'destination', 'airbyte/destination-pubsub', '0.1.0', 'other', '{"type": "SPEC"}'), 81 | ('Snowflake', 'destination', 'airbyte/destination-snowflake', '0.3.11', 'snowflake', '{"type": "SPEC"}'), 82 | ('S3', 'destination', 'airbyte/destination-s3', '0.1.9', 'other', '{"type": "SPEC"}'), 83 | ('Redshift', 'destination', 'airbyte/destination-redshift', '0.3.12', 'redshift', '{"type": "SPEC"}'), 84 | ('MeiliSearch', 'destination', 'airbyte/destination-meilisearch', '0.2.8', 'other', '{"type": "SPEC"}'), 85 | ('MySQL', 'destination', 'airbyte/destination-mysql', '0.1.9', 'mysql', '{"type": "SPEC"}'), 86 | ('MS SQL Server', 'destination', 'airbyte/destination-mssql', '0.1.6', 'other', '{"type": "SPEC"}'), 87 | ('Oracle (Alpha)', 'destination', 'airbyte/destination-oracle', '0.1.3', 'other', '{"type": "SPEC"}'), 88 | ('Kafka', 'destination', 'airbyte/destination-kafka', '0.1.0', 'other', '{"type": "SPEC"}') 89 | ; 90 | 91 | INSERT INTO connectors(name, type, docker_image_name, docker_image_tag, destination_type, spec) VALUES 92 | ('Amazon Seller Partner', 'source', 'airbyte/source-amazon-seller-partner', '0.1.3', '', '{"type": "SPEC"}'), 93 | ('Asana', 'source', 'airbyte/source-asana', '0.1.1', '', '{"type": "SPEC"}'), 94 | ('Exchange Rates Api', 'source', 'airbyte/source-exchange-rates', '0.2.3', '', '{"type": "SPEC"}'), 95 | ('File', 'source', 'airbyte/source-file', '0.2.4', '', '{"type": "SPEC"}'), 96 | ('Google Ads', 'source', 'airbyte/source-google-ads', '0.1.2', '', '{"type": "SPEC"}'), 97 | ('Google Adwords (Deprecated)', 'source', 'airbyte/source-google-adwords-singer', '0.2.6', '', '{"type": "SPEC"}'), 98 | ('GitHub', 'source', 'airbyte/source-github', '0.1.2', '', '{"type": "SPEC"}'), 99 | ('Microsoft SQL Server (MSSQL)', 'source', 'airbyte/source-mssql', '0.3.3', '', '{"type": "SPEC"}'), 100 | ('Pipedrive', 'source', 'airbyte/source-pipedrive', '0.1.0', '', '{"type": "SPEC"}'), 101 | ('Postgres', 'source', 'airbyte/source-postgres', '0.3.7', '', '{"type": "SPEC"}'), 102 | ('Cockroachdb', 'source', 'airbyte/source-cockroachdb', '0.1.1', '', '{"type": "SPEC"}'), 103 | ('PostHog', 'source', 'airbyte/source-posthog', '0.1.2', '', '{"type": "SPEC"}'), 104 | ('Recurly', 'source', 'airbyte/source-recurly', '0.2.4', '', '{"type": "SPEC"}'), 105 | ('Sendgrid', 'source', 'airbyte/source-sendgrid', '0.2.6', '', '{"type": "SPEC"}'), 106 | ('Marketo', 'source', 'airbyte/source-marketo-singer', '0.2.3', '', '{"type": "SPEC"}'), 107 | ('Google Sheets', 'source', 'airbyte/source-google-sheets', '0.2.3', '', '{"type": "SPEC"}'), 108 | ('MySQL', 'source', 'airbyte/source-mysql', '0.4.0', '', '{"type": "SPEC"}'), 109 | ('Salesforce', 'source', 'airbyte/source-salesforce-singer', '0.2.4', '', '{"type": "SPEC"}'), 110 | ('Stripe', 'source', 'airbyte/source-stripe', '0.1.14', '', '{"type": "SPEC"}'), 111 | ('Mailchimp', 'source', 'airbyte/source-mailchimp', '0.2.5', '', '{"type": "SPEC"}'), 112 | ('Google Analytics', 'source', 'airbyte/source-googleanalytics-singer', '0.2.6', '', '{"type": "SPEC"}'), 113 | ('Facebook Marketing', 'source', 'airbyte/source-facebook-marketing', '0.2.14', '', '{"type": "SPEC"}'), 114 | ('Hubspot', 'source', 'airbyte/source-hubspot', '0.1.5', '', '{"type": "SPEC"}'), 115 | ('Klaviyo', 'source', 'airbyte/source-klaviyo', '0.1.1', '', '{"type": "SPEC"}'), 116 | ('Shopify', 'source', 'airbyte/source-shopify', '0.1.10', '', '{"type": "SPEC"}'), 117 | ('HTTP Request', 'source', 'airbyte/source-http-request', '0.2.4', '', '{"type": "SPEC"}'), 118 | ('Redshift', 'source', 'airbyte/source-redshift', '0.3.1', '', '{"type": "SPEC"}'), 119 | ('Twilio', 'source', 'airbyte/source-twilio', '0.1.0', '', '{"type": "SPEC"}'), 120 | ('Freshdesk', 'source', 'airbyte/source-freshdesk', '0.2.5', '', '{"type": "SPEC"}'), 121 | ('Braintree', 'source', 'airbyte/source-braintree-singer', '0.2.3', '', '{"type": "SPEC"}'), 122 | ('Greenhouse', 'source', 'airbyte/source-greenhouse', '0.2.3', '', '{"type": "SPEC"}'), 123 | ('Zendesk Chat', 'source', 'airbyte/source-zendesk-chat', '0.1.1', '', '{"type": "SPEC"}'), 124 | ('Zendesk Support', 'source', 'airbyte/source-zendesk-support-singer', '0.2.3', '', '{"type": "SPEC"}'), 125 | ('Intercom', 'source', 'airbyte/source-intercom', '0.1.0', '', '{"type": "SPEC"}'), 126 | ('Jira', 'source', 'airbyte/source-jira', '0.2.7', '', '{"type": "SPEC"}'), 127 | ('Mixpanel', 'source', 'airbyte/source-mixpanel', '0.1.0', '', '{"type": "SPEC"}'), 128 | ('Mixpanel Singer', 'source', 'airbyte/source-mixpanel-singer', '0.2.4', '', '{"type": "SPEC"}'), 129 | ('Zoom', 'source', 'airbyte/source-zoom-singer', '0.2.4', '', '{"type": "SPEC"}'), 130 | ('Microsoft teams', 'source', 'airbyte/source-microsoft-teams', '0.2.2', '', '{"type": "SPEC"}'), 131 | ('Drift', 'source', 'airbyte/source-drift', '0.2.2', '', '{"type": "SPEC"}'), 132 | ('Looker', 'source', 'airbyte/source-looker', '0.2.4', '', '{"type": "SPEC"}'), 133 | ('Plaid', 'source', 'airbyte/source-plaid', '0.2.1', '', '{"type": "SPEC"}'), 134 | ('Appstore', 'source', 'airbyte/source-appstore-singer', '0.2.4', '', '{"type": "SPEC"}'), 135 | ('Mongo DB', 'source', 'airbyte/source-mongodb', '0.3.3', '', '{"type": "SPEC"}'), 136 | ('Google Directory', 'source', 'airbyte/source-google-directory', '0.1.3', '', '{"type": "SPEC"}'), 137 | ('Instagram', 'source', 'airbyte/source-instagram', '0.1.7', '', '{"type": "SPEC"}'), 138 | ('Gitlab', 'source', 'airbyte/source-gitlab', '0.1.0', '', '{"type": "SPEC"}'), 139 | ('Google Workspace Admin Reports', 'source', 'airbyte/source-google-workspace-admin-reports', '0.1.4', '', '{"type": "SPEC"}'), 140 | ('Tempo', 'source', 'airbyte/source-tempo', '0.2.3', '', '{"type": "SPEC"}'), 141 | ('Smartsheets', 'source', 'airbyte/source-smartsheets', '0.1.5', '', '{"type": "SPEC"}'), 142 | ('Oracle DB', 'source', 'airbyte/source-oracle', '0.3.1', '', '{"type": "SPEC"}'), 143 | ('Zendesk Talk', 'source', 'airbyte/source-zendesk-talk', '0.1.2', '', '{"type": "SPEC"}'), 144 | ('Quickbooks', 'source', 'airbyte/source-quickbooks-singer', '0.1.2', '', '{"type": "SPEC"}'), 145 | ('Iterable', 'source', 'airbyte/source-iterable', '0.1.6', '', '{"type": "SPEC"}'), 146 | ('PokeAPI', 'source', 'airbyte/source-pokeapi', '0.1.1', '', '{"type": "SPEC"}'), 147 | ('Google Search Console', 'source', 'airbyte/source-google-search-console-singer', '0.1.3', '', '{"type": "SPEC"}'), 148 | ('ClickHouse', 'source', 'airbyte/source-clickhouse', '0.1.1', '', '{"type": "SPEC"}'), 149 | ('Recharge', 'source', 'airbyte/source-recharge', '0.1.1', '', '{"type": "SPEC"}'), 150 | ('Harvest', 'source', 'airbyte/source-harvest', '0.1.3', '', '{"type": "SPEC"}'), 151 | ('Amplitude', 'source', 'airbyte/source-amplitude', '0.1.1', '', '{"type": "SPEC"}'), 152 | ('Snowflake', 'source', 'airbyte/source-snowflake', '0.1.0', '', '{"type": "SPEC"}'), 153 | ('IBM Db2', 'source', 'airbyte/source-db2', '0.1.0', '', '{"type": "SPEC"}'), 154 | ('Slack', 'source', 'airbyte/source-slack', '0.1.8', '', '{"type": "SPEC"}'), 155 | ('AWS CloudTrail', 'source', 'airbyte/source-aws-cloudtrail', '0.1.1', '', '{"type": "SPEC"}'), 156 | ('US Census', 'source', 'airbyte/source-us-census', '0.1.0', '', '{"type": "SPEC"}'), 157 | ('Okta', 'source', 'airbyte/source-okta', '0.1.2', '', '{"type": "SPEC"}'), 158 | ('Survey Monkey', 'source', 'airbyte/source-surveymonkey', '0.1.0', '', '{"type": "SPEC"}'), 159 | ('Square', 'source', 'airbyte/source-square', '0.1.1', '', '{"type": "SPEC"}'), 160 | ('Zendesk Sunshine', 'source', 'airbyte/source-zendesk-sunshine', '0.1.0', '', '{"type": "SPEC"}'), 161 | ('Paypal Transaction', 'source', 'airbyte/source-paypal-transaction', '0.1.0', '', '{"type": "SPEC"}'), 162 | ('Dixa', 'source', 'airbyte/source-dixa', '0.1.0', '', '{"type": "SPEC"}'), 163 | ('Typeform', 'source', 'airbyte/source-typeform', '0.1.0', '', '{"type": "SPEC"}') 164 | ; 165 | --------------------------------------------------------------------------------