├── docs ├── static │ ├── .nojekyll │ ├── img │ │ └── favicon.ico │ └── users │ │ ├── gojek.png │ │ ├── goto.png │ │ ├── jago.png │ │ ├── mapan.png │ │ ├── moka.png │ │ ├── midtrans.png │ │ └── paylater.png ├── docs │ ├── roadmap.md │ ├── guides │ │ ├── 4_clients.md │ │ ├── 2_manage_namespace.md │ │ └── 0_introduction.md │ ├── formats │ │ ├── avro.md │ │ ├── json.md │ │ └── protobuf.md │ ├── usecases.md │ ├── clients │ │ └── overview.md │ └── glossary.md ├── babel.config.js ├── blog │ ├── authors.yml │ └── 2021-08-20-stenciil-launch.md ├── .gitignore ├── README.md ├── src │ ├── core │ │ ├── Container.js │ │ └── GridBlock.js │ ├── css │ │ └── theme.css │ └── pages │ │ └── help.js ├── package.json └── sidebars.js ├── ui ├── src │ ├── App.css │ ├── react-app-env.d.ts │ ├── App.tsx │ ├── setupTests.ts │ ├── App.test.tsx │ ├── index.css │ ├── reportWebVitals.ts │ ├── index.tsx │ └── api.ts ├── public │ ├── favicon.ico │ ├── logo192.png │ ├── logo512.png │ ├── robots.txt │ ├── manifest.json │ └── index.html ├── Makefile ├── embed.go ├── .gitignore ├── tsconfig.json ├── package.json └── README.md ├── clients ├── python │ ├── src │ │ └── raystack │ │ │ ├── __init__.py │ │ │ ├── store.py │ │ │ └── stencil.py │ ├── requirements.txt │ ├── conftest.py │ ├── test │ │ ├── data │ │ │ └── one.proto │ │ └── test_client.py │ ├── .gitignore │ ├── setup.py │ └── README.md ├── js │ ├── .gitignore │ ├── main.js │ ├── jest.config.js │ ├── .eslintrc.yml │ ├── test │ │ └── data │ │ │ └── one.proto │ ├── package.json │ └── lib │ │ ├── refresh_strategy.js │ │ └── multi_url_stencil.js ├── java │ ├── lombok.config │ ├── gradle │ │ └── wrapper │ │ │ ├── gradle-wrapper.jar │ │ │ └── gradle-wrapper.properties │ ├── .gitignore │ ├── src │ │ ├── test │ │ │ ├── proto │ │ │ │ ├── ProtoWithoutJavaPackage.proto │ │ │ │ ├── RecursiveProtoMessage.proto │ │ │ │ ├── ProtoWithoutPackage.proto │ │ │ │ ├── TestMessage.proto │ │ │ │ └── NestedProtoMessage.proto │ │ │ └── java │ │ │ │ └── io │ │ │ │ └── odpf │ │ │ │ └── stencil │ │ │ │ ├── client │ │ │ │ └── ClassLoadStencilClientTest.java │ │ │ │ ├── URLStencilClientTest.java │ │ │ │ ├── MultiURLStencilClientTest.java │ │ │ │ └── http │ │ │ │ └── RetryHttpClientTest.java │ │ └── main │ │ │ └── java │ │ │ └── io │ │ │ └── odpf │ │ │ └── stencil │ │ │ ├── exception │ │ │ └── StencilRuntimeException.java │ │ │ ├── http │ │ │ ├── RemoteFile.java │ │ │ ├── RemoteFileImpl.java │ │ │ └── RetryHttpClient.java │ │ │ ├── SchemaUpdateListener.java │ │ │ ├── Parser.java │ │ │ ├── client │ │ │ ├── StencilClient.java │ │ │ ├── ClassLoadStencilClient.java │ │ │ └── MultiURLStencilClient.java │ │ │ ├── StencilClientFactory.java │ │ │ └── cache │ │ │ └── SchemaRefreshStrategy.java │ └── settings.gradle ├── go │ ├── test_data │ │ ├── 2.proto │ │ ├── 3.proto │ │ ├── extension.proto │ │ ├── 1.proto │ │ └── updated │ │ │ └── 1.proto │ ├── go.mod │ ├── logger.go │ ├── downloader.go │ ├── protoutils.go │ ├── go.sum │ └── store.go ├── clojure │ ├── test │ │ └── stencil │ │ │ ├── testdata │ │ │ ├── testdata.desc │ │ │ └── one.proto │ │ │ └── core_test.clj │ ├── .gitignore │ ├── project.clj │ └── src │ │ └── stencil │ │ └── decode.clj └── README.md ├── internal ├── store │ ├── postgres │ │ ├── migrations │ │ │ ├── 000002_create_search_data_idx.down.sql │ │ │ ├── 000002_create_search_data_idx.up.sql │ │ │ ├── 000001_initialize_schema.down.sql │ │ │ └── 000001_initialize_schema.up.sql │ │ ├── postgres_test.go │ │ └── postgres.go │ └── errors.go ├── api │ ├── testdata │ │ └── test.desc │ ├── ping.go │ ├── api_test.go │ ├── search.go │ └── mocks │ │ └── search_service.go └── server │ ├── newrelic.go │ └── graceful.go ├── config ├── build.go ├── load.go ├── config.go └── config.yaml ├── formats ├── protobuf │ ├── testdata │ │ ├── backward │ │ │ ├── current │ │ │ │ ├── 2.proto │ │ │ │ └── 1.proto │ │ │ └── previous │ │ │ │ ├── 2.proto │ │ │ │ └── 1.proto │ │ ├── valid │ │ │ └── 1.proto │ │ ├── compatible │ │ │ ├── previous │ │ │ │ └── 1.proto │ │ │ └── current │ │ │ │ └── 1.proto │ │ └── forward │ │ │ ├── current │ │ │ └── 1.proto │ │ │ └── previous │ │ │ └── 1.proto │ ├── schema_test.go │ ├── error.go │ ├── provider.go │ └── schema.go ├── json │ ├── testdata │ │ ├── propertyAddition │ │ │ ├── removed.json │ │ │ ├── prev.json │ │ │ └── added.json │ │ ├── enum │ │ │ ├── non_enum.json │ │ │ ├── curr_removal.json │ │ │ ├── prev.json │ │ │ └── curr_addition.json │ │ ├── allOf │ │ │ ├── deleted.json │ │ │ ├── prev.json │ │ │ ├── noChange.json │ │ │ └── modified.json │ │ ├── anyOf │ │ │ ├── deleted.json │ │ │ ├── prev.json │ │ │ ├── noChange.json │ │ │ └── modified.json │ │ ├── oneOf │ │ │ ├── deleted.json │ │ │ ├── prev.json │ │ │ ├── noChange.json │ │ │ └── modified.json │ │ ├── refChange │ │ │ ├── prev.json │ │ │ ├── removed.json │ │ │ └── modified.json │ │ ├── array │ │ │ ├── 2020items.json │ │ │ ├── draft7additionalItems.json │ │ │ ├── 2020prev.json │ │ │ ├── 2020updated.json │ │ │ ├── draft7prev.json │ │ │ ├── draft7updated.json │ │ │ ├── 2020prefixItems.json │ │ │ └── draft7items.json │ │ ├── propertyDeleted │ │ │ ├── modifiedSchema.json │ │ │ └── prevSchema.json │ │ ├── additionalProperties │ │ │ ├── openContent.json │ │ │ └── partialOpenContent.json │ │ ├── typeChecks │ │ │ └── typeCheckSchema.json │ │ └── requiredProperties │ │ │ ├── removed.json │ │ │ ├── prev.json │ │ │ ├── added.json │ │ │ └── modified.json │ ├── utils_test.go │ ├── provider.go │ ├── error.go │ ├── provider_test.go │ └── schema.go └── avro │ ├── provider.go │ └── schema.go ├── Dockerfile ├── docker-compose.yml ├── pkg ├── logger │ └── logger.go └── graph │ └── graph.go ├── main.go ├── core ├── schema │ ├── utils.go │ ├── provider │ │ └── provider.go │ ├── mocks │ │ ├── schema_provider.go │ │ ├── namespace_service.go │ │ └── schema_cache.go │ ├── schema.go │ └── compatibility.go ├── namespace │ ├── namespace.go │ └── service.go └── search │ ├── search.go │ └── service.go ├── buf.gen.yaml ├── .gitignore ├── .github ├── workflows │ ├── test-java-client.yml │ ├── release-js-client.yml │ ├── test-go-client.yml │ ├── test-js-client.yml │ ├── test-clojure-client.yml │ ├── docs.yml │ ├── lint.yml │ ├── release-server.yml │ ├── release-java-client.yml │ └── test-server.yaml └── ISSUE_TEMPLATE │ ├── feature_request.md │ └── bug_report.md ├── cmd ├── cdk.go ├── help.go ├── errors.go ├── config.go ├── download.go ├── edit.go ├── client.go ├── root.go ├── delete.go ├── server.go ├── schema.go ├── graph.go ├── check.go └── list.go ├── .golangci.yaml ├── Makefile └── .goreleaser.yaml /docs/static/.nojekyll: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ui/src/App.css: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /docs/docs/roadmap.md: -------------------------------------------------------------------------------- 1 | # Roadmap 2 | -------------------------------------------------------------------------------- /clients/python/src/raystack/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /clients/js/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | coverage 3 | -------------------------------------------------------------------------------- /docs/docs/guides/4_clients.md: -------------------------------------------------------------------------------- 1 | # Using clients 2 | -------------------------------------------------------------------------------- /ui/src/react-app-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | -------------------------------------------------------------------------------- /ui/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/stencil/HEAD/ui/public/favicon.ico -------------------------------------------------------------------------------- /ui/public/logo192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/stencil/HEAD/ui/public/logo192.png -------------------------------------------------------------------------------- /ui/public/logo512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/stencil/HEAD/ui/public/logo512.png -------------------------------------------------------------------------------- /ui/public/robots.txt: -------------------------------------------------------------------------------- 1 | # https://www.robotstxt.org/robotstxt.html 2 | User-agent: * 3 | Disallow: 4 | -------------------------------------------------------------------------------- /docs/static/img/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/stencil/HEAD/docs/static/img/favicon.ico -------------------------------------------------------------------------------- /docs/static/users/gojek.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/stencil/HEAD/docs/static/users/gojek.png -------------------------------------------------------------------------------- /docs/static/users/goto.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/stencil/HEAD/docs/static/users/goto.png -------------------------------------------------------------------------------- /docs/static/users/jago.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/stencil/HEAD/docs/static/users/jago.png -------------------------------------------------------------------------------- /docs/static/users/mapan.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/stencil/HEAD/docs/static/users/mapan.png -------------------------------------------------------------------------------- /docs/static/users/moka.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/stencil/HEAD/docs/static/users/moka.png -------------------------------------------------------------------------------- /docs/static/users/midtrans.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/stencil/HEAD/docs/static/users/midtrans.png -------------------------------------------------------------------------------- /docs/static/users/paylater.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/stencil/HEAD/docs/static/users/paylater.png -------------------------------------------------------------------------------- /internal/store/postgres/migrations/000002_create_search_data_idx.down.sql: -------------------------------------------------------------------------------- 1 | DROP INDEX IF EXISTS search_data_idx; -------------------------------------------------------------------------------- /internal/api/testdata/test.desc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/stencil/HEAD/internal/api/testdata/test.desc -------------------------------------------------------------------------------- /ui/src/App.tsx: -------------------------------------------------------------------------------- 1 | function App() { 2 | return ( 3 |
hello
4 | ); 5 | } 6 | 7 | export default App; 8 | -------------------------------------------------------------------------------- /docs/babel.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | presets: [require.resolve('@docusaurus/core/lib/babel/preset')], 3 | }; 4 | -------------------------------------------------------------------------------- /clients/java/lombok.config: -------------------------------------------------------------------------------- 1 | # This file is generated by the 'io.freefair.lombok' Gradle plugin 2 | config.stopBubbling = true 3 | -------------------------------------------------------------------------------- /config/build.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | var ( 4 | Version = "dev" 5 | BuildCommit = "" 6 | BuildDate = "" 7 | ) 8 | -------------------------------------------------------------------------------- /formats/protobuf/testdata/backward/current/2.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package a; 4 | 5 | message Test { 6 | 7 | } 8 | -------------------------------------------------------------------------------- /clients/go/test_data/2.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package test; 4 | 5 | message Three { 6 | int64 field_one = 1; 7 | } 8 | -------------------------------------------------------------------------------- /formats/protobuf/testdata/backward/previous/2.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package a; 4 | 5 | message Test { 6 | 7 | } 8 | -------------------------------------------------------------------------------- /clients/python/requirements.txt: -------------------------------------------------------------------------------- 1 | protobuf==3.17.3 2 | pytest==6.2.5 3 | pytest-cov==2.12.1 4 | schedule==1.1.0 5 | requests==2.26.0 6 | -------------------------------------------------------------------------------- /formats/json/testdata/propertyAddition/removed.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "summaries", 3 | "properties": { 4 | 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /internal/store/postgres/migrations/000002_create_search_data_idx.up.sql: -------------------------------------------------------------------------------- 1 | CREATE INDEX search_data_idx ON schema_files USING gin (search_data); -------------------------------------------------------------------------------- /ui/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: dist test 2 | 3 | dist: 4 | @yarn run build 5 | 6 | test: 7 | @yarn run test 8 | 9 | dep: 10 | @yarn install -------------------------------------------------------------------------------- /ui/embed.go: -------------------------------------------------------------------------------- 1 | package ui 2 | 3 | import "embed" 4 | 5 | // Assets embed build folder 6 | // 7 | //go:embed build/* 8 | var Assets embed.FS 9 | -------------------------------------------------------------------------------- /clients/go/test_data/3.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | option java_package = "test.stencil"; 3 | message Root { 4 | int64 field_one = 1; 5 | } 6 | -------------------------------------------------------------------------------- /clients/java/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/stencil/HEAD/clients/java/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /docs/docs/formats/avro.md: -------------------------------------------------------------------------------- 1 | # Avro 2 | 3 | We are in the process of actively improving Stencil documentation. This guide will be updated very soon. 4 | -------------------------------------------------------------------------------- /docs/docs/formats/json.md: -------------------------------------------------------------------------------- 1 | # JSON 2 | 3 | We are in the process of actively improving Stencil documentation. This guide will be updated very soon. 4 | -------------------------------------------------------------------------------- /clients/clojure/test/stencil/testdata/testdata.desc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/stencil/HEAD/clients/clojure/test/stencil/testdata/testdata.desc -------------------------------------------------------------------------------- /docs/docs/formats/protobuf.md: -------------------------------------------------------------------------------- 1 | # Protobuf 2 | 3 | We are in the process of actively improving Stencil documentation. This guide will be updated very soon. 4 | -------------------------------------------------------------------------------- /docs/blog/authors.yml: -------------------------------------------------------------------------------- 1 | ravisuhag: 2 | name: Ravi Suhag 3 | title: Maintainer 4 | url: https://github.com/ravisuhag 5 | image_url: https://github.com/ravisuhag.png 6 | -------------------------------------------------------------------------------- /docs/docs/guides/2_manage_namespace.md: -------------------------------------------------------------------------------- 1 | # Manage namespaces 2 | 3 | We are in the process of actively improving Stencil documentation. This guide will be updated very soon. 4 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.13 2 | 3 | RUN apk add --no-cache ca-certificates && update-ca-certificates 4 | 5 | COPY stencil /usr/bin/stencil 6 | 7 | EXPOSE 8080 8 | ENTRYPOINT ["stencil"] 9 | -------------------------------------------------------------------------------- /clients/js/main.js: -------------------------------------------------------------------------------- 1 | const Stencil = require('./lib/stencil'); 2 | const MultiURLStencil = require('./lib/multi_url_stencil'); 3 | 4 | module.exports = { 5 | Stencil, 6 | MultiURLStencil 7 | }; 8 | -------------------------------------------------------------------------------- /formats/json/testdata/enum/non_enum.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "summaries", 3 | "properties": { 4 | "roles": { 5 | "title": "Organization roles" 6 | } 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /clients/clojure/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /classes 3 | /checkouts 4 | profiles.clj 5 | pom.xml 6 | pom.xml.asc 7 | *.jar 8 | *.class 9 | /.lein-* 10 | /.nrepl-port 11 | /.prepl-port 12 | .hgignore 13 | .hg/ 14 | -------------------------------------------------------------------------------- /formats/protobuf/testdata/valid/1.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package a; 3 | import "google/protobuf/duration.proto"; 4 | 5 | message Test { 6 | string field1 = 1; 7 | google.protobuf.Duration field2 = 2; 8 | } 9 | -------------------------------------------------------------------------------- /clients/go/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/raystack/stencil/clients/go 2 | 3 | go 1.16 4 | 5 | require ( 6 | github.com/pkg/errors v0.9.1 7 | github.com/stretchr/testify v1.7.0 8 | google.golang.org/protobuf v1.26.0 9 | ) 10 | -------------------------------------------------------------------------------- /clients/js/jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | clearMocks: true, 3 | collectCoverage: true, 4 | coverageThreshold: { 5 | global: { 6 | statements: 95 7 | } 8 | }, 9 | testEnvironment: 'node' 10 | }; 11 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | db: 4 | image: "postgres:13" 5 | ports: 6 | - "5432:5432" 7 | environment: 8 | POSTGRES_DB: "stencil_dev" 9 | POSTGRES_HOST_AUTH_METHOD: "trust" 10 | -------------------------------------------------------------------------------- /formats/json/testdata/allOf/deleted.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "summaries", 3 | "properties": { 4 | "roles": { 5 | "title": "Organization roles", 6 | "type": "object" 7 | } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /formats/json/testdata/anyOf/deleted.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "summaries", 3 | "properties": { 4 | "roles": { 5 | "title": "Organization roles", 6 | "type": "object" 7 | } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /formats/json/testdata/oneOf/deleted.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "summaries", 3 | "properties": { 4 | "roles": { 5 | "title": "Organization roles", 6 | "type": "object" 7 | } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /formats/json/testdata/propertyAddition/prev.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "summaries", 3 | "properties": { 4 | "roles": { 5 | "title": "Organization roles", 6 | "type": "string" 7 | } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /clients/java/.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | out 3 | .gradle 4 | *.iml 5 | .DS_Store 6 | 7 | build 8 | src/test/generated/ 9 | src/test/resources/ 10 | classpath 11 | 12 | .classpath 13 | .vscode/ 14 | .project 15 | .settings/ 16 | bin/ 17 | private.key 18 | -------------------------------------------------------------------------------- /internal/store/postgres/migrations/000001_initialize_schema.down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS versions_schema_files; 2 | DROP TABLE IF EXISTS schema_files; 3 | DROP TABLE IF EXISTS versions; 4 | DROP TABLE IF EXISTS schemas; 5 | DROP TABLE IF EXISTS namespaces; 6 | -------------------------------------------------------------------------------- /clients/java/src/test/proto/ProtoWithoutJavaPackage.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package org.raystack.stencil; 4 | 5 | message ImplicitOuterClass { 6 | string sample_string = 1; 7 | message Inner { 8 | string one = 1; 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /clients/java/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-6.8.3-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /docs/blog/2021-08-20-stenciil-launch.md: -------------------------------------------------------------------------------- 1 | --- 2 | slug: introducing-stencil 3 | title: Introducing Stencil 4 | authors: 5 | name: Ravi Suhag 6 | title: Maintainer 7 | url: https://github.com/ravisuhag 8 | tags: [raystack, stencil] 9 | --- 10 | 11 | We are live! 12 | -------------------------------------------------------------------------------- /ui/src/setupTests.ts: -------------------------------------------------------------------------------- 1 | // jest-dom adds custom jest matchers for asserting on DOM nodes. 2 | // allows you to do things like: 3 | // expect(element).toHaveTextContent(/react/i) 4 | // learn more: https://github.com/testing-library/jest-dom 5 | import '@testing-library/jest-dom'; 6 | -------------------------------------------------------------------------------- /clients/java/src/main/java/io/odpf/stencil/exception/StencilRuntimeException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.stencil.exception; 2 | 3 | public class StencilRuntimeException extends RuntimeException{ 4 | public StencilRuntimeException(Throwable t) { 5 | super(t); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /clients/java/src/main/java/io/odpf/stencil/http/RemoteFile.java: -------------------------------------------------------------------------------- 1 | package org.raystack.stencil.http; 2 | 3 | import java.io.IOException; 4 | 5 | public interface RemoteFile { 6 | byte[] fetch(String url) throws IOException; 7 | void close() throws IOException; 8 | } 9 | 10 | -------------------------------------------------------------------------------- /clients/java/src/main/java/io/odpf/stencil/SchemaUpdateListener.java: -------------------------------------------------------------------------------- 1 | package org.raystack.stencil; 2 | 3 | import java.util.Map; 4 | import com.google.protobuf.Descriptors; 5 | 6 | public interface SchemaUpdateListener { 7 | void onSchemaUpdate(final Map newDescriptor); 8 | } 9 | -------------------------------------------------------------------------------- /pkg/logger/logger.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "log" 5 | 6 | "go.uber.org/zap" 7 | ) 8 | 9 | // Logger zap logger instance 10 | var Logger *zap.Logger 11 | 12 | func init() { 13 | l, err := zap.NewProduction() 14 | if err != nil { 15 | log.Fatalln(err) 16 | } 17 | Logger = l 18 | } 19 | -------------------------------------------------------------------------------- /clients/java/src/main/java/io/odpf/stencil/Parser.java: -------------------------------------------------------------------------------- 1 | package org.raystack.stencil; 2 | 3 | import com.google.protobuf.DynamicMessage; 4 | import com.google.protobuf.InvalidProtocolBufferException; 5 | 6 | public interface Parser { 7 | DynamicMessage parse(byte[] data) throws InvalidProtocolBufferException; 8 | } 9 | -------------------------------------------------------------------------------- /formats/json/testdata/refChange/prev.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "summaries", 3 | "properties": { 4 | "roles": { 5 | "title": "Organization roles", 6 | "type": "string" 7 | }, 8 | "roleRef": { 9 | "$ref": "#/properties/roles" 10 | } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /docs/.gitignore: -------------------------------------------------------------------------------- 1 | # Dependencies 2 | /node_modules 3 | 4 | # Production 5 | /build 6 | 7 | # Generated files 8 | .docusaurus 9 | .cache-loader 10 | 11 | # Misc 12 | .DS_Store 13 | .env.local 14 | .env.development.local 15 | .env.test.local 16 | .env.production.local 17 | 18 | npm-debug.log* 19 | yarn-debug.log* 20 | yarn-error.log* -------------------------------------------------------------------------------- /formats/json/testdata/propertyAddition/added.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "summaries", 3 | "properties": { 4 | "roles": { 5 | "title": "Organization roles", 6 | "type": "string" 7 | }, 8 | "roleRef": { 9 | "$ref": "#/properties/roles" 10 | } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /ui/src/App.test.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { render, screen } from '@testing-library/react'; 3 | import App from './App'; 4 | 5 | test('renders learn react link', () => { 6 | // render(); 7 | // const linkElement = screen.getByText(/learn react/i); 8 | // expect(linkElement).toBeInTheDocument(); 9 | }); 10 | -------------------------------------------------------------------------------- /clients/go/test_data/extension.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package test; 4 | 5 | message ExtendableMessage { 6 | optional int64 field_extra = 2; 7 | extensions 10 to 20; 8 | } 9 | 10 | message Extender { 11 | required int64 field_one = 1; 12 | extend ExtendableMessage { 13 | optional string field_two = 11; 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /formats/json/testdata/enum/curr_removal.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "summaries", 3 | "properties": { 4 | "roles": { 5 | "title": "Organization roles", 6 | "type": "string", 7 | "enum": [ 8 | "producer", 9 | "licensor" 10 | ] 11 | } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /config/load.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "github.com/raystack/salt/config" 5 | ) 6 | 7 | func Load(configFile string) (Config, error) { 8 | var cfg Config 9 | loader := config.NewLoader(config.WithFile(configFile)) 10 | 11 | if err := loader.Load(&cfg); err != nil { 12 | return Config{}, err 13 | } 14 | 15 | return cfg, nil 16 | } 17 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/raystack/stencil/cmd" 8 | ) 9 | 10 | const ( 11 | exitOK = 0 12 | exitError = 1 13 | ) 14 | 15 | func main() { 16 | command := cmd.New() 17 | 18 | if err := command.Execute(); err != nil { 19 | fmt.Fprintln(os.Stderr, err) 20 | os.Exit(exitError) 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /clients/js/.eslintrc.yml: -------------------------------------------------------------------------------- 1 | env: 2 | browser: false 3 | commonjs: true 4 | es2021: true 5 | jest: true 6 | extends: 7 | - airbnb-base 8 | - prettier 9 | parserOptions: 10 | ecmaVersion: 12 11 | rules: 12 | quotes: ["error", "single", { "avoidEscape": true }] 13 | no-console: ["error", allow: ["error"]] 14 | semi: 15 | - error 16 | - always 17 | -------------------------------------------------------------------------------- /formats/json/testdata/refChange/removed.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "summaries", 3 | "properties": { 4 | "roles": { 5 | "title": "Organization roles", 6 | "type": "string" 7 | }, 8 | "roleRef": { 9 | "title": "Organization roles reference", 10 | "type": "string" 11 | } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /formats/json/testdata/allOf/prev.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "summaries", 3 | "properties": { 4 | "roles": { 5 | "title": "Organization roles", 6 | "type": "object", 7 | "allOf": [ 8 | { 9 | "$ref": "http://json-schema.org/draft-07/schema" 10 | } 11 | ] 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /formats/json/testdata/anyOf/prev.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "summaries", 3 | "properties": { 4 | "roles": { 5 | "title": "Organization roles", 6 | "type": "object", 7 | "anyOf": [ 8 | { 9 | "$ref": "http://json-schema.org/draft-07/schema" 10 | } 11 | ] 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /formats/json/testdata/oneOf/prev.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "summaries", 3 | "properties": { 4 | "roles": { 5 | "title": "Organization roles", 6 | "type": "object", 7 | "oneOf": [ 8 | { 9 | "$ref": "http://json-schema.org/draft-07/schema" 10 | } 11 | ] 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /formats/json/testdata/allOf/noChange.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "summaries", 3 | "properties": { 4 | "roles": { 5 | "title": "Organization roles", 6 | "type": "object", 7 | "allOf": [ 8 | { 9 | "$ref": "http://json-schema.org/draft-07/schema" 10 | } 11 | ] 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /formats/json/testdata/anyOf/noChange.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "summaries", 3 | "properties": { 4 | "roles": { 5 | "title": "Organization roles", 6 | "type": "object", 7 | "anyOf": [ 8 | { 9 | "$ref": "http://json-schema.org/draft-07/schema" 10 | } 11 | ] 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /formats/json/testdata/oneOf/noChange.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "summaries", 3 | "properties": { 4 | "roles": { 5 | "title": "Organization roles", 6 | "type": "object", 7 | "oneOf": [ 8 | { 9 | "$ref": "http://json-schema.org/draft-07/schema" 10 | } 11 | ] 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /formats/json/testdata/enum/prev.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "summaries", 3 | "properties": { 4 | "roles": { 5 | "title": "Organization roles", 6 | "type": "string", 7 | "enum": [ 8 | "producer", 9 | "licensor", 10 | "processor", 11 | "host" 12 | ] 13 | } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /internal/api/ping.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "context" 5 | 6 | "google.golang.org/grpc/health/grpc_health_v1" 7 | ) 8 | 9 | // Check grpc health check 10 | func (s *API) Check(ctx context.Context, in *grpc_health_v1.HealthCheckRequest) (*grpc_health_v1.HealthCheckResponse, error) { 11 | return &grpc_health_v1.HealthCheckResponse{Status: grpc_health_v1.HealthCheckResponse_SERVING}, nil 12 | } 13 | -------------------------------------------------------------------------------- /clients/java/settings.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * This file was generated by the Gradle 'init' task. 3 | * 4 | * The settings file is used to specify which projects to include in your build. 5 | * 6 | * Detailed information about configuring a multi-project build in Gradle can be found 7 | * in the user guide at https://docs.gradle.org/4.6/userguide/multi_project_builds.html 8 | */ 9 | 10 | rootProject.name = 'stencil' 11 | -------------------------------------------------------------------------------- /clients/java/src/test/proto/RecursiveProtoMessage.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package org.raystack.stencil; 4 | 5 | option java_multiple_files = true; 6 | option java_package = "org.raystack.stencil"; 7 | option java_outer_classname = "Recursive"; 8 | 9 | 10 | message RecursiveLogMessage { 11 | string id = 1; 12 | message RECORD { 13 | string id = 1; 14 | RECORD record = 2; 15 | } 16 | } -------------------------------------------------------------------------------- /ui/.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | 8 | # testing 9 | /coverage 10 | 11 | # production 12 | /build 13 | 14 | # misc 15 | .DS_Store 16 | .env.local 17 | .env.development.local 18 | .env.test.local 19 | .env.production.local 20 | 21 | npm-debug.log* 22 | yarn-debug.log* 23 | yarn-error.log* 24 | -------------------------------------------------------------------------------- /ui/src/index.css: -------------------------------------------------------------------------------- 1 | body { 2 | margin: 0; 3 | font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', 4 | 'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue', 5 | sans-serif; 6 | -webkit-font-smoothing: antialiased; 7 | -moz-osx-font-smoothing: grayscale; 8 | } 9 | 10 | code { 11 | font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New', 12 | monospace; 13 | } 14 | -------------------------------------------------------------------------------- /formats/json/testdata/enum/curr_addition.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "summaries", 3 | "properties": { 4 | "roles": { 5 | "title": "Organization roles", 6 | "type": "string", 7 | "enum": [ 8 | "producer", 9 | "licensor", 10 | "processor", 11 | "host", 12 | "super_user" 13 | ] 14 | } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /core/schema/utils.go: -------------------------------------------------------------------------------- 1 | package schema 2 | 3 | import "fmt" 4 | 5 | func getNonEmpty(args ...string) string { 6 | for _, a := range args { 7 | if a != "" { 8 | return a 9 | } 10 | } 11 | return "" 12 | } 13 | 14 | func schemaKeyFunc(nsName, schema string, version int32) string { 15 | return fmt.Sprintf("%s-%s-%d", nsName, schema, version) 16 | } 17 | 18 | func getBytes(key interface{}) []byte { 19 | buf, _ := key.([]byte) 20 | return buf 21 | } 22 | -------------------------------------------------------------------------------- /formats/json/testdata/refChange/modified.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "summaries", 3 | "properties": { 4 | "roles": { 5 | "title": "Organization roles", 6 | "type": "string" 7 | }, 8 | "doubleRole": { 9 | "title": "Organization Double roles", 10 | "type": "string" 11 | }, 12 | "roleRef": { 13 | "$ref": "#/properties/doubleRole" 14 | } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /clients/java/src/test/proto/ProtoWithoutPackage.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | option java_package = "org.raystack.stencil"; 4 | option java_multiple_files = true; 5 | option java_outer_classname = "ProtoWithoutPackage"; 6 | 7 | message RootField { 8 | string string_field = 1; 9 | NestedField nested_field = 2; 10 | uint32 int_field = 3; 11 | } 12 | 13 | message NestedField { 14 | string string_field = 1; 15 | uint32 int_field = 2; 16 | } 17 | -------------------------------------------------------------------------------- /buf.gen.yaml: -------------------------------------------------------------------------------- 1 | version: v1 2 | plugins: 3 | - name: go 4 | out: ./proto 5 | opt: paths=source_relative 6 | - name: go-grpc 7 | out: ./proto 8 | opt: paths=source_relative 9 | - remote: buf.build/raystack/plugins/validate 10 | out: "proto" 11 | opt: "paths=source_relative,lang=go" 12 | - name: grpc-gateway 13 | out: ./proto 14 | opt: paths=source_relative 15 | - name: openapiv2 16 | out: ./proto 17 | opt: "allow_merge=true" 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | .DS_Store 8 | 9 | # Test binary, built with `go test -c` 10 | *.test 11 | 12 | # Output of the go coverage tool, specifically when used with LiteIDE 13 | *.out 14 | 15 | # Dependency directories 16 | vendor/ 17 | 18 | # IDEs 19 | .idea 20 | .vscode 21 | 22 | # Project specific ignore 23 | .env 24 | !*/stencil 25 | /stencil 26 | /config.yaml 27 | node_modules 28 | 29 | __debug* 30 | example 31 | 32 | -------------------------------------------------------------------------------- /clients/java/src/test/proto/TestMessage.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package org.raystack.stencil; 4 | 5 | option java_multiple_files = true; 6 | option java_package = "org.raystack.stencil"; 7 | option java_outer_classname = "TestMessageProto"; 8 | 9 | message TestKey { 10 | string sample_string = 1; 11 | } 12 | 13 | message TestMessage { 14 | string sample_string = 1; 15 | } 16 | 17 | message TestMessageSuperset { 18 | string sample_string = 1; 19 | bool success = 9; 20 | } 21 | -------------------------------------------------------------------------------- /ui/src/reportWebVitals.ts: -------------------------------------------------------------------------------- 1 | import { ReportHandler } from 'web-vitals'; 2 | 3 | const reportWebVitals = (onPerfEntry?: ReportHandler) => { 4 | if (onPerfEntry && onPerfEntry instanceof Function) { 5 | import('web-vitals').then(({ getCLS, getFID, getFCP, getLCP, getTTFB }) => { 6 | getCLS(onPerfEntry); 7 | getFID(onPerfEntry); 8 | getFCP(onPerfEntry); 9 | getLCP(onPerfEntry); 10 | getTTFB(onPerfEntry); 11 | }); 12 | } 13 | }; 14 | 15 | export default reportWebVitals; 16 | -------------------------------------------------------------------------------- /clients/python/conftest.py: -------------------------------------------------------------------------------- 1 | from subprocess import run 2 | 3 | import pytest 4 | import os 5 | 6 | 7 | @pytest.fixture(scope="session") 8 | def protoc_setup(): 9 | current_dir = os.path.dirname(os.path.realpath(__file__)) 10 | output_file = os.path.join(current_dir, 'test/data/one.desc') 11 | 12 | input_dir = os.path.join(current_dir, 'test/data') 13 | run(['protoc', f'--descriptor_set_out={output_file}', '--include_imports', f'--proto_path={input_dir}', 14 | 'one.proto'], cwd=current_dir) 15 | -------------------------------------------------------------------------------- /formats/json/utils_test.go: -------------------------------------------------------------------------------- 1 | package json 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/santhosh-tekuri/jsonschema/v5" 7 | "github.com/stretchr/testify/assert" 8 | 9 | _ "github.com/santhosh-tekuri/jsonschema/v5/httploader" 10 | ) 11 | 12 | func TestExploreJsonSchemaRecursively(t *testing.T) { 13 | sc, err := jsonschema.Compile("testdata/collection.json") 14 | assert.Nil(t, err) 15 | exploredMap := exploreSchema(sc) 16 | assert.NotEmpty(t, exploredMap) 17 | assert.Equal(t, 46, len(exploredMap)) 18 | } 19 | -------------------------------------------------------------------------------- /formats/json/testdata/allOf/modified.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "summaries", 3 | "properties": { 4 | "roles": { 5 | "title": "Organization roles", 6 | "type": "object", 7 | "allOf": [ 8 | { 9 | "$ref": "http://json-schema.org/draft-07/schema" 10 | }, 11 | { 12 | "$ref": "https://json-schema.org/draft/2020-12/schema" 13 | } 14 | ] 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /formats/json/testdata/anyOf/modified.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "summaries", 3 | "properties": { 4 | "roles": { 5 | "title": "Organization roles", 6 | "type": "object", 7 | "anyOf": [ 8 | { 9 | "$ref": "http://json-schema.org/draft-07/schema" 10 | }, 11 | { 12 | "$ref": "https://json-schema.org/draft/2020-12/schema" 13 | } 14 | ] 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /formats/json/testdata/oneOf/modified.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "summaries", 3 | "properties": { 4 | "roles": { 5 | "title": "Organization roles", 6 | "type": "object", 7 | "oneOf": [ 8 | { 9 | "$ref": "http://json-schema.org/draft-07/schema" 10 | }, 11 | { 12 | "$ref": "https://json-schema.org/draft/2020-12/schema" 13 | } 14 | ] 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /formats/protobuf/testdata/compatible/previous/1.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package a; 4 | 5 | enum TestEnum { 6 | UNSPECIFIED = 0; 7 | ONE = 1; 8 | } 9 | // Unchange message 10 | message One { 11 | string field_one = 1; 12 | } 13 | 14 | // Backward compatible message 15 | message Two { 16 | reserved 4,5; 17 | reserved "deleted_field"; 18 | string string_field = 1; 19 | int64 int_field = 2; 20 | TestEnum enum_field = 3; 21 | enum CompatibleEnum { 22 | UNKNOWN = 0; 23 | ONE = 1; 24 | } 25 | } 26 | 27 | 28 | -------------------------------------------------------------------------------- /clients/clojure/project.clj: -------------------------------------------------------------------------------- 1 | (defproject org.raystack/stencil-clj "0.5.1" 2 | :description "Stencil client for clojure" 3 | :url "https://github.com/raystack/stencil" 4 | :license {:name "Apache 2.0" 5 | :url "https://www.apache.org/licenses/LICENSE-2.0"} 6 | :dependencies [[org.clojure/clojure "1.10.3"] 7 | [org.raystack/stencil "0.4.0"]] 8 | :plugins [[lein-cljfmt "0.7.0"]] 9 | :global-vars {*warn-on-reflection* true} 10 | :source-paths ["src"] 11 | :repl-options {:init-ns stencil.core}) 12 | -------------------------------------------------------------------------------- /clients/go/test_data/1.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package test; 4 | import "google/protobuf/timestamp.proto"; 5 | option java_package = "test.stencil"; 6 | 7 | message One { 8 | int64 field_one = 1; 9 | } 10 | 11 | message Two { 12 | message Three { 13 | string data = 1; 14 | google.protobuf.Timestamp timestamp = 3; 15 | } 16 | Three id = 1; 17 | message Four { 18 | Four recursive = 2; 19 | string field_two = 3; 20 | message Five { 21 | double id = 1; 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /clients/go/logger.go: -------------------------------------------------------------------------------- 1 | package stencil 2 | 3 | // Logger interface used to get logging from stencil internals. 4 | type Logger interface { 5 | Info(string) 6 | Error(string) 7 | } 8 | 9 | type wrappedLogger struct { 10 | l Logger 11 | } 12 | 13 | func (w wrappedLogger) Info(msg string) { 14 | if w.l != nil { 15 | w.l.Info(msg) 16 | } 17 | } 18 | 19 | func (w wrappedLogger) Error(msg string) { 20 | if w.l != nil { 21 | w.l.Info(msg) 22 | } 23 | } 24 | 25 | func wrapLogger(l Logger) Logger { 26 | return wrappedLogger{l} 27 | } 28 | -------------------------------------------------------------------------------- /clients/js/test/data/one.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package test; 4 | import "google/protobuf/timestamp.proto"; 5 | option java_package = "test.stencil"; 6 | 7 | message One { 8 | int64 field_one = 1; 9 | } 10 | 11 | message Two { 12 | message Three { 13 | string data = 1; 14 | google.protobuf.Timestamp timestamp = 3; 15 | } 16 | Three id = 1; 17 | message Four { 18 | Four recursive = 2; 19 | string field_two = 3; 20 | message Five { 21 | double id = 1; 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /clients/python/test/data/one.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package test; 4 | import "google/protobuf/timestamp.proto"; 5 | option java_package = "test.stencil"; 6 | 7 | message One { 8 | int64 field_one = 1; 9 | } 10 | 11 | message Two { 12 | message Three { 13 | string data = 1; 14 | google.protobuf.Timestamp timestamp = 3; 15 | } 16 | Three id = 1; 17 | message Four { 18 | Four recursive = 2; 19 | string field_two = 3; 20 | message Five { 21 | double id = 1; 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /formats/avro/provider.go: -------------------------------------------------------------------------------- 1 | package avro 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" 7 | av "github.com/hamba/avro" 8 | "github.com/raystack/stencil/core/schema" 9 | ) 10 | 11 | // ParseSchema parses avro schema bytes into ParsedSchema 12 | func ParseSchema(data []byte) (schema.ParsedSchema, error) { 13 | sc, err := av.Parse(string(data)) 14 | if err != nil { 15 | return nil, &runtime.HTTPStatusError{HTTPStatus: http.StatusBadRequest, Err: err} 16 | } 17 | return &Schema{sc: sc, data: data}, nil 18 | } 19 | -------------------------------------------------------------------------------- /clients/go/test_data/updated/1.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package test; 4 | import "google/protobuf/timestamp.proto"; 5 | option java_package = "test.stencil"; 6 | 7 | message One { 8 | int64 field_one = 1; 9 | int64 field_two = 2; 10 | } 11 | 12 | message Two { 13 | message Three { 14 | string data = 1; 15 | google.protobuf.Timestamp timestamp = 3; 16 | } 17 | Three id = 1; 18 | message Four { 19 | Four recursive = 2; 20 | string field_two = 3; 21 | message Five { 22 | double id = 1; 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /formats/protobuf/testdata/compatible/current/1.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package a; 4 | 5 | enum TestEnum { 6 | UNSPECIFIED = 0; 7 | ONE = 1; 8 | } 9 | // Unchange message 10 | message One { 11 | string field_one = 1; 12 | } 13 | 14 | // Backward compatible message 15 | message Two { 16 | reserved 4,5; 17 | reserved "deleted_field"; 18 | string string_field = 1; 19 | int64 int_field = 2; 20 | TestEnum enum_field = 3; 21 | One new_field = 6; 22 | enum CompatibleEnum { 23 | UNKNOWN = 0; 24 | ONE = 1; 25 | TWO = 2; 26 | } 27 | } 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /internal/server/newrelic.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "log" 5 | 6 | "github.com/newrelic/go-agent/v3/newrelic" 7 | "github.com/raystack/stencil/config" 8 | ) 9 | 10 | func getNewRelic(config *config.Config) *newrelic.Application { 11 | newRelicConfig := config.NewRelic 12 | app, err := newrelic.NewApplication( 13 | newrelic.ConfigAppName(newRelicConfig.AppName), 14 | newrelic.ConfigLicense(newRelicConfig.License), 15 | newrelic.ConfigEnabled(newRelicConfig.Enabled), 16 | ) 17 | if err != nil { 18 | log.Fatal(err) 19 | } 20 | return app 21 | } 22 | -------------------------------------------------------------------------------- /internal/store/postgres/postgres_test.go: -------------------------------------------------------------------------------- 1 | package postgres_test 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/raystack/stencil/internal/store/postgres" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func tearDown(t *testing.T) { 12 | t.Helper() 13 | connectionString := os.Getenv("TEST_DB_CONNECTIONSTRING") 14 | if connectionString == "" { 15 | t.Skip("Skipping test since DB info not available") 16 | return 17 | } 18 | m, err := postgres.NewHTTPFSMigrator(connectionString) 19 | if assert.NoError(t, err) { 20 | m.Down() 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /formats/json/testdata/array/2020items.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://json-schema.org/draft/2020-12/schema", 3 | "title": "STAC Collection Specification", 4 | "description": "This object represents Collections in a SpatioTemporal Asset Catalog.", 5 | "properties": { 6 | "example": { 7 | "type": "array", 8 | "prefixItems": [ 9 | { 10 | "type":"string" 11 | }, 12 | { 13 | "type": "number" 14 | } 15 | ] 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /formats/json/testdata/array/draft7additionalItems.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-07/schema#", 3 | "title": "STAC Collection Specification", 4 | "description": "This object represents Collections in a SpatioTemporal Asset Catalog.", 5 | "properties": { 6 | "example": { 7 | "type": "array", 8 | "items": [ 9 | { 10 | "type":"string" 11 | }, 12 | { 13 | "type": "number" 14 | } 15 | ] 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /core/namespace/namespace.go: -------------------------------------------------------------------------------- 1 | package namespace 2 | 3 | import ( 4 | "context" 5 | "time" 6 | ) 7 | 8 | type Namespace struct { 9 | ID string 10 | Format string 11 | Compatibility string 12 | Description string 13 | CreatedAt time.Time 14 | UpdatedAt time.Time 15 | } 16 | 17 | type Repository interface { 18 | Create(context.Context, Namespace) (Namespace, error) 19 | Update(context.Context, Namespace) (Namespace, error) 20 | List(context.Context) ([]Namespace, error) 21 | Get(context.Context, string) (Namespace, error) 22 | Delete(context.Context, string) error 23 | } 24 | -------------------------------------------------------------------------------- /.github/workflows/test-java-client.yml: -------------------------------------------------------------------------------- 1 | name: Test stencil clients 2 | on: 3 | push: 4 | paths: 5 | - "clients/java/**" 6 | branches: 7 | - main 8 | pull_request: 9 | paths: 10 | - "clients/java/**" 11 | branches: 12 | - main 13 | jobs: 14 | test-java: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - uses: actions/checkout@v2 18 | - name: Set up JDK 8 19 | uses: actions/setup-java@v2 20 | with: 21 | distribution: adopt 22 | java-version: 8 23 | - name: Run tests 24 | run: cd clients/java/ && ./gradlew test 25 | -------------------------------------------------------------------------------- /ui/public/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "short_name": "React App", 3 | "name": "Create React App Sample", 4 | "icons": [ 5 | { 6 | "src": "favicon.ico", 7 | "sizes": "64x64 32x32 24x24 16x16", 8 | "type": "image/x-icon" 9 | }, 10 | { 11 | "src": "logo192.png", 12 | "type": "image/png", 13 | "sizes": "192x192" 14 | }, 15 | { 16 | "src": "logo512.png", 17 | "type": "image/png", 18 | "sizes": "512x512" 19 | } 20 | ], 21 | "start_url": ".", 22 | "display": "standalone", 23 | "theme_color": "#000000", 24 | "background_color": "#ffffff" 25 | } 26 | -------------------------------------------------------------------------------- /ui/src/index.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ReactDOM from 'react-dom/client'; 3 | import App from './App'; 4 | import reportWebVitals from './reportWebVitals'; 5 | 6 | const root = ReactDOM.createRoot( 7 | document.getElementById('root') as HTMLElement 8 | ); 9 | root.render( 10 | 11 | 12 | 13 | ); 14 | 15 | // If you want to start measuring performance in your app, pass a function 16 | // to log results (for example: reportWebVitals(console.log)) 17 | // or send to an analytics endpoint. Learn more: https://bit.ly/CRA-vitals 18 | reportWebVitals(); 19 | -------------------------------------------------------------------------------- /cmd/cdk.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | var dict = map[string]string{ 4 | "COMPATIBILITY_BACKWARD": "backward", 5 | "COMPATIBILITY_FORWARD": "forward", 6 | "COMPATIBILITY_FULL": "full", 7 | "COMPATIBILITY_UNSPECIFIED": "-", 8 | "FORMAT_PROTOBUF": "protobuf", 9 | "FORMAT_JSON": "json", 10 | "FORMAT_AVRO": "avro", 11 | } 12 | 13 | var ( 14 | formats = []string{ 15 | "FORMAT_JSON", 16 | "FORMAT_PROTOBUF", 17 | "FORMAT_AVRO", 18 | } 19 | 20 | comps = []string{ 21 | "COMPATIBILITY_BACKWARD", 22 | "COMPATIBILITY_FORWARD", 23 | "COMPATIBILITY_FULL", 24 | } 25 | ) 26 | -------------------------------------------------------------------------------- /ui/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es5", 4 | "lib": [ 5 | "dom", 6 | "dom.iterable", 7 | "esnext" 8 | ], 9 | "allowJs": true, 10 | "skipLibCheck": true, 11 | "esModuleInterop": true, 12 | "allowSyntheticDefaultImports": true, 13 | "strict": true, 14 | "forceConsistentCasingInFileNames": true, 15 | "noFallthroughCasesInSwitch": true, 16 | "module": "esnext", 17 | "moduleResolution": "node", 18 | "resolveJsonModule": true, 19 | "isolatedModules": true, 20 | "noEmit": true, 21 | "jsx": "react-jsx" 22 | }, 23 | "include": [ 24 | "src" 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /core/search/search.go: -------------------------------------------------------------------------------- 1 | package search 2 | 3 | import "context" 4 | 5 | type Repository interface { 6 | Search(context.Context, *SearchRequest) ([]*SearchHits, error) 7 | SearchLatest(context.Context, *SearchRequest) ([]*SearchHits, error) 8 | } 9 | 10 | type SearchRequest struct { 11 | NamespaceID string 12 | SchemaID string 13 | Query string 14 | History bool 15 | VersionID int32 16 | } 17 | 18 | type SearchResponse struct { 19 | Hits []*SearchHits 20 | } 21 | 22 | type SearchHits struct { 23 | Fields []string 24 | Types []string 25 | Path string 26 | NamespaceID string 27 | SchemaID string 28 | VersionID int32 29 | } 30 | -------------------------------------------------------------------------------- /formats/json/testdata/array/2020prev.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://json-schema.org/draft/2020-12/schema", 3 | "title": "STAC Collection Specification", 4 | "description": "This object represents Collections in a SpatioTemporal Asset Catalog.", 5 | "properties": { 6 | "example": { 7 | "type": "array", 8 | "prefixItems": [ 9 | { 10 | "type":"string" 11 | }, 12 | { 13 | "type": "number" 14 | } 15 | ], 16 | "items": { 17 | "type":"integer" 18 | } 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /formats/json/testdata/array/2020updated.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://json-schema.org/draft/2020-12/schema", 3 | "title": "STAC Collection Specification", 4 | "description": "This object represents Collections in a SpatioTemporal Asset Catalog.", 5 | "properties": { 6 | "example": { 7 | "type": "array", 8 | "prefixItems": [ 9 | { 10 | "type":"string" 11 | }, 12 | { 13 | "type": "string" 14 | } 15 | ], 16 | "items": { 17 | "type":"integer" 18 | } 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /formats/json/testdata/array/draft7prev.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-07/schema#", 3 | "title": "STAC Collection Specification", 4 | "description": "This object represents Collections in a SpatioTemporal Asset Catalog.", 5 | "properties": { 6 | "example": { 7 | "type": "array", 8 | "items": [ 9 | { 10 | "type":"string" 11 | }, 12 | { 13 | "type": "number" 14 | } 15 | ], 16 | "additionalItems": { 17 | "type":"integer" 18 | } 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /formats/json/testdata/array/draft7updated.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-07/schema#", 3 | "title": "STAC Collection Specification", 4 | "description": "This object represents Collections in a SpatioTemporal Asset Catalog.", 5 | "properties": { 6 | "example": { 7 | "type": "array", 8 | "items": [ 9 | { 10 | "type":"string" 11 | }, 12 | { 13 | "type": "string" 14 | } 15 | ], 16 | "additionalItems": { 17 | "type":"integer" 18 | } 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /ui/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | React App 13 | 14 | 15 | 16 | 17 |
18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /clients/java/src/test/proto/NestedProtoMessage.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package org.raystack.stencil; 4 | 5 | option java_multiple_files = true; 6 | option java_package = "org.raystack.stencil"; 7 | option java_outer_classname = "AccountDbAccounts"; 8 | 9 | 10 | message account_db_accounts { 11 | message ID { 12 | string data = 1; 13 | } 14 | ID id = 1; 15 | string operationtype = 2; 16 | string clustertime = 3; 17 | message FULLDOCUMENT { 18 | string id = 1; 19 | string cif = 2; 20 | string customerid = 3; 21 | message ACCOUNTS_ITEM { 22 | double monthlyaveragebalance = 1; 23 | } 24 | } 25 | } -------------------------------------------------------------------------------- /.github/workflows/release-js-client.yml: -------------------------------------------------------------------------------- 1 | name: Release Stencil JS Client 2 | on: 3 | push: 4 | tags: 5 | - "v*.*.*" 6 | workflow_dispatch: 7 | 8 | jobs: 9 | publish-js-client: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v2 13 | - uses: actions/setup-node@v2 14 | with: 15 | node-version: "12.x" 16 | registry-url: "https://registry.npmjs.org" 17 | scope: "@raystack" 18 | - run: npm install 19 | working-directory: clients/js 20 | - run: npm publish --access public 21 | working-directory: clients/js 22 | env: 23 | NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} 24 | -------------------------------------------------------------------------------- /ui/src/api.ts: -------------------------------------------------------------------------------- 1 | 2 | export const getNamespaces = () => fetch("/v1beta1/namespaces").then(res => res.json()); 3 | export const getSchemas = (namespace : string) => fetch(`/v1beta1/namespaces/${namespace}/schemas`).then(res => res.json()); 4 | export const getVersions = (namespace: string, schema : string) => fetch(`/v1beta1/namespaces/${namespace}/schemas/${schema}/versions`).then(res => res.json()); 5 | export const getLatestSchema = (namespace: string, schema : string) => fetch(`/v1beta1/namespaces/${namespace}/schemas/${schema}`); 6 | export const getVersionedSchema = (namespace: string, schema : string, version : number) => fetch(`/v1beta1/namespaces/${namespace}/schemas/${schema}/versions/${version}`) 7 | -------------------------------------------------------------------------------- /internal/api/api_test.go: -------------------------------------------------------------------------------- 1 | package api_test 2 | 3 | import ( 4 | "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" 5 | "github.com/raystack/stencil/internal/api" 6 | "github.com/raystack/stencil/internal/api/mocks" 7 | ) 8 | 9 | func setup() (*mocks.NamespaceService, *mocks.SchemaService, *mocks.SearchService, *runtime.ServeMux, *api.API) { 10 | nsService := &mocks.NamespaceService{} 11 | schemaService := &mocks.SchemaService{} 12 | searchService := &mocks.SearchService{} 13 | mux := runtime.NewServeMux() 14 | v1beta1 := api.NewAPI(nsService, schemaService, searchService) 15 | v1beta1.RegisterSchemaHandlers(mux, nil) 16 | return nsService, schemaService, searchService, mux, v1beta1 17 | } 18 | -------------------------------------------------------------------------------- /.golangci.yaml: -------------------------------------------------------------------------------- 1 | run: 2 | timeout: 5m 3 | output: 4 | formats: 5 | - format: colored-line-number 6 | linters: 7 | enable-all: false 8 | disable-all: true 9 | enable: 10 | - govet 11 | - goimports 12 | - thelper 13 | - tparallel 14 | - unconvert 15 | - wastedassign 16 | - revive 17 | - unused 18 | - gofmt 19 | - whitespace 20 | - misspell 21 | linters-settings: 22 | revive: 23 | ignore-generated-header: true 24 | severity: warning 25 | issues: 26 | exclude-dirs: 27 | - api/proto 28 | - clients/java 29 | - clients/js 30 | - docs 31 | - scripts 32 | - ui 33 | fix: true 34 | severity: 35 | default-severity: error 36 | -------------------------------------------------------------------------------- /cmd/help.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/MakeNowJust/heredoc" 5 | "github.com/raystack/salt/cli/commander" 6 | ) 7 | 8 | var envHelpTopics = []commander.HelpTopic{ 9 | { 10 | Name: "environment", 11 | Short: "List of supported environment variables", 12 | Long: heredoc.Doc(` 13 | RAYSTACK_CONFIG_DIR: the directory where stencil will store configuration files. Default: 14 | "$XDG_CONFIG_HOME/raystack" or "$HOME/.config/raystack". 15 | 16 | NO_COLOR: set to any value to avoid printing ANSI escape sequences for color output. 17 | 18 | CLICOLOR: set to "0" to disable printing ANSI colors in output. 19 | `), 20 | }, 21 | } 22 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /formats/json/testdata/array/2020prefixItems.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://json-schema.org/draft/2020-12/schema", 3 | "title": "STAC Collection Specification", 4 | "description": "This object represents Collections in a SpatioTemporal Asset Catalog.", 5 | "properties": { 6 | "example": { 7 | "type": "array", 8 | "prefixItems": [ 9 | { 10 | "type":"string" 11 | }, 12 | { 13 | "type": "number" 14 | }, 15 | { 16 | "type":"string" 17 | } 18 | ], 19 | "items": { 20 | "type":"integer" 21 | } 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /formats/json/testdata/array/draft7items.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-07/schema#", 3 | "title": "STAC Collection Specification", 4 | "description": "This object represents Collections in a SpatioTemporal Asset Catalog.", 5 | "properties": { 6 | "example": { 7 | "type": "array", 8 | "items": [ 9 | { 10 | "type":"string" 11 | }, 12 | { 13 | "type": "number" 14 | }, 15 | { 16 | "type": "string" 17 | } 18 | ], 19 | "additionalItems": { 20 | "type":"integer" 21 | } 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /.github/workflows/test-go-client.yml: -------------------------------------------------------------------------------- 1 | name: Test stencil GO client 2 | on: 3 | push: 4 | paths: 5 | - "clients/go/**" 6 | branches: 7 | - main 8 | pull_request: 9 | paths: 10 | - "clients/go/**" 11 | branches: 12 | - main 13 | jobs: 14 | test-go: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - name: Set up Go 1.x 18 | uses: actions/setup-go@v2 19 | with: 20 | go-version: ^1.16 21 | id: go 22 | - name: Install Protoc 23 | uses: arduino/setup-protoc@v1 24 | with: 25 | version: '3.x' 26 | - name: Check out code into the Go module directory 27 | uses: actions/checkout@v2 28 | - name: Test 29 | run: cd clients/go; go test -count 1 -cover ./... 30 | -------------------------------------------------------------------------------- /clients/python/.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | bin/ 10 | build/ 11 | develop-eggs/ 12 | dist/ 13 | eggs/ 14 | lib/ 15 | lib64/ 16 | parts/ 17 | sdist/ 18 | var/ 19 | *.egg-info/ 20 | .installed.cfg 21 | *.egg 22 | 23 | # Installer logs 24 | pip-log.txt 25 | pip-delete-this-directory.txt 26 | 27 | # Unit test / coverage reports 28 | .tox/ 29 | .coverage 30 | .cache 31 | nosetests.xml 32 | coverage.xml 33 | 34 | # Translations 35 | *.mo 36 | 37 | # Mr Developer 38 | .mr.developer.cfg 39 | .project 40 | .pydevproject 41 | 42 | # Rope 43 | .ropeproject 44 | 45 | # Django stuff: 46 | *.log 47 | *.pot 48 | 49 | # Sphinx documentation 50 | docs/_build/ 51 | *.desc 52 | -------------------------------------------------------------------------------- /clients/python/src/raystack/store.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from google.protobuf.descriptor_pb2 import FileDescriptorSet 3 | from google.protobuf.message import Message 4 | from google.protobuf.message_factory import GetMessages 5 | 6 | 7 | class Store: 8 | def __init__(self): 9 | self.data = {} 10 | 11 | def get(self, name) -> Message: 12 | return self.data.get(name) 13 | 14 | def _load_from_url(self, url): 15 | result = requests.get(url, stream=True) 16 | return result.raw.read() 17 | 18 | def load(self, url: str = None, data: bytes = None): 19 | if url: 20 | data = self._load_from_url(url) 21 | fds = FileDescriptorSet.FromString(data) 22 | messages = GetMessages([file for file in fds.file]) 23 | self.data.update(messages) 24 | -------------------------------------------------------------------------------- /clients/go/downloader.go: -------------------------------------------------------------------------------- 1 | package stencil 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "net/http" 7 | ) 8 | 9 | func downloader(uri string, opts HTTPOptions) ([]byte, error) { 10 | req, err := http.NewRequest("GET", uri, nil) 11 | if err != nil { 12 | return nil, fmt.Errorf("invalid request. %w", err) 13 | } 14 | for key, val := range opts.Headers { 15 | req.Header.Add(key, val) 16 | } 17 | res, err := (&http.Client{Timeout: opts.Timeout}).Do(req) 18 | if err != nil { 19 | return nil, fmt.Errorf("request failed. %w", err) 20 | } 21 | defer res.Body.Close() 22 | switch res.StatusCode { 23 | case 200: 24 | return ioutil.ReadAll(res.Body) 25 | default: 26 | body, err := ioutil.ReadAll(res.Body) 27 | return nil, fmt.Errorf("request failed. response body: %s, response_read_error: %w", body, err) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /clients/clojure/test/stencil/core_test.clj: -------------------------------------------------------------------------------- 1 | (ns stencil.core-test 2 | (:require [clojure.test :refer :all] 3 | [stencil.core :refer :all]) 4 | (:import 5 | (org.raystack.stencil.client StencilClient))) 6 | 7 | (deftest test-create-client 8 | (testing "should create client" 9 | (let [config {:url "http://localhost:8000/v1beta1/namespaces/raystack/schemas/proton" 10 | :refresh-ttl 100 11 | :request-timeout 10000 12 | :request-backoff-time 100 13 | :retry-count 3 14 | :refresh-cache true 15 | :headers {"Authorization" "Bearer token"} 16 | :refresh-strategy :long-polling-refresh} 17 | client (create-client config)] 18 | (is (instance? StencilClient client))))) 19 | -------------------------------------------------------------------------------- /cmd/errors.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/MakeNowJust/heredoc" 7 | ) 8 | 9 | var ( 10 | ErrClientConfigNotFound = errors.New(heredoc.Doc(` 11 | Stencil client config not found. 12 | Run "stencil config init" to initialize a new client config or 13 | Run "stencil help environment" for more information. 14 | `)) 15 | ErrClientConfigHostNotFound = errors.New(heredoc.Doc(` 16 | Stencil client config "host" not found. 17 | Pass stencil server host with "--host" flag or 18 | set host in stencil config. 19 | Run "stencil config " or 20 | "stencil help environment" for more information. 21 | `)) 22 | ErrClientNotAuthorized = errors.New(heredoc.Doc(` 23 | Stencil auth error. Stencil requires an auth header. 24 | 25 | Run "stencil help auth" for more information. 26 | `)) 27 | ) 28 | -------------------------------------------------------------------------------- /formats/json/provider.go: -------------------------------------------------------------------------------- 1 | package json 2 | 3 | import ( 4 | js "encoding/json" 5 | "net/http" 6 | 7 | "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" 8 | "github.com/raystack/stencil/core/schema" 9 | "github.com/santhosh-tekuri/jsonschema/v5" 10 | ) 11 | 12 | func GetParsedSchema(data []byte) (schema.ParsedSchema, error) { 13 | compiler := jsonschema.NewCompiler() 14 | compiler.Draft = jsonschema.Draft2020 15 | sc, _ := compiler.Compile("https://json-schema.org/draft/2020-12/schema") 16 | var val interface{} 17 | if err := js.Unmarshal(data, &val); err != nil { 18 | return nil, &runtime.HTTPStatusError{HTTPStatus: http.StatusBadRequest, Err: err} 19 | } 20 | if err := sc.Validate(val); err != nil { 21 | return nil, &runtime.HTTPStatusError{HTTPStatus: http.StatusBadRequest, Err: err} 22 | } 23 | return &Schema{data: data}, nil 24 | } 25 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # Website 2 | 3 | This website is built using [Docusaurus 2](https://docusaurus.io/), a modern static website generator. 4 | 5 | ### Installation 6 | 7 | ``` 8 | $ yarn 9 | ``` 10 | 11 | ### Local Development 12 | 13 | ``` 14 | $ yarn start 15 | ``` 16 | 17 | This command starts a local development server and opens up a browser window. Most changes are reflected live without having to restart the server. 18 | 19 | ### Build 20 | 21 | ``` 22 | $ yarn build 23 | ``` 24 | 25 | This command generates static content into the `build` directory and can be served using any static contents hosting service. 26 | 27 | ### Deployment 28 | 29 | ``` 30 | $ GIT_USER= USE_SSH=true yarn deploy 31 | ``` 32 | 33 | If you are using GitHub pages for hosting, this command is a convenient way to build the website and push to the `gh-pages` branch. 34 | -------------------------------------------------------------------------------- /.github/workflows/test-js-client.yml: -------------------------------------------------------------------------------- 1 | name: Test stencil JS client 2 | on: 3 | push: 4 | paths: 5 | - "clients/js/**" 6 | branches: 7 | - main 8 | pull_request: 9 | paths: 10 | - "clients/js/**" 11 | branches: 12 | - main 13 | jobs: 14 | test: 15 | runs-on: ubuntu-latest 16 | strategy: 17 | matrix: 18 | node-version: ['12', '14'] 19 | steps: 20 | - uses: actions/checkout@v2 21 | - uses: actions/setup-node@v2 22 | with: 23 | node-version: ${{ matrix.node-version }} 24 | - name: Install Protoc 25 | uses: arduino/setup-protoc@v1 26 | with: 27 | version: '3.x' 28 | - name: Install dependencies 29 | run: npm ci 30 | working-directory: clients/js 31 | - name: Test 32 | run: npm test 33 | working-directory: clients/js 34 | -------------------------------------------------------------------------------- /core/namespace/service.go: -------------------------------------------------------------------------------- 1 | package namespace 2 | 3 | import ( 4 | "context" 5 | ) 6 | 7 | type Service struct { 8 | repo Repository 9 | } 10 | 11 | func NewService(repository Repository) *Service { 12 | return &Service{ 13 | repo: repository, 14 | } 15 | } 16 | 17 | func (s Service) Create(ctx context.Context, ns Namespace) (Namespace, error) { 18 | return s.repo.Create(ctx, ns) 19 | } 20 | 21 | func (s Service) Update(ctx context.Context, ns Namespace) (Namespace, error) { 22 | return s.repo.Update(ctx, ns) 23 | } 24 | 25 | func (s Service) List(ctx context.Context) ([]Namespace, error) { 26 | return s.repo.List(ctx) 27 | } 28 | 29 | func (s Service) Get(ctx context.Context, name string) (Namespace, error) { 30 | return s.repo.Get(ctx, name) 31 | } 32 | 33 | func (s Service) Delete(ctx context.Context, name string) error { 34 | return s.repo.Delete(ctx, name) 35 | } 36 | -------------------------------------------------------------------------------- /formats/json/testdata/propertyDeleted/modifiedSchema.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "summaries", 3 | "properties": { 4 | "minimum": { 5 | "title": "Minimum value", 6 | "type": [ 7 | "number", 8 | "string" 9 | ] 10 | }, 11 | "maximum": { 12 | "title": "Maximum value", 13 | "type": [ 14 | "number" 15 | ] 16 | }, 17 | "setValue": { 18 | "title": "Set of values", 19 | "type": "array", 20 | "minItems": 1, 21 | "items": { 22 | "description": "For each field only the original data type of the property can occur (except for arrays), but we can't validate that in JSON Schema yet. See the sumamry description in the STAC specification for details." 23 | } 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import "time" 4 | 5 | // NewRelicConfig contains the New Relic go-agent configuration 6 | type NewRelicConfig struct { 7 | Enabled bool `default:"false"` 8 | AppName string `default:"stencil"` 9 | License string 10 | } 11 | 12 | // DBConfig contains DB connection details 13 | type DBConfig struct { 14 | ConnectionString string 15 | } 16 | 17 | // GRPCConfig grpc options 18 | type GRPCConfig struct { 19 | MaxRecvMsgSizeInMB int `default:"10"` 20 | MaxSendMsgSizeInMB int `default:"10"` 21 | } 22 | 23 | // Config Server config 24 | type Config struct { 25 | Port string `default:"8080"` 26 | // Timeout represents graceful shutdown period. Defaults to 60 seconds. 27 | Timeout time.Duration `default:"60s"` 28 | CacheSizeInMB int64 `default:"100"` 29 | GRPC GRPCConfig 30 | NewRelic NewRelicConfig 31 | DB DBConfig 32 | } 33 | -------------------------------------------------------------------------------- /formats/json/testdata/propertyDeleted/prevSchema.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "summaries", 3 | "properties": { 4 | "minimum": { 5 | "title": "Minimum value", 6 | "type": [ 7 | "number", 8 | "string" 9 | ] 10 | }, 11 | "maximum": { 12 | "title": "Maximum value", 13 | "type": [ 14 | "number", 15 | "string" 16 | ] 17 | }, 18 | "setValue": { 19 | "title": "Set of values", 20 | "type": "array", 21 | "minItems": 1, 22 | "items": { 23 | "description": "For each field only the original data type of the property can occur (except for arrays), but we can't validate that in JSON Schema yet. See the sumamry description in the STAC specification for details." 24 | } 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /formats/json/testdata/additionalProperties/openContent.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "summaries", 3 | "properties": { 4 | "minimum": { 5 | "title": "Minimum value", 6 | "type": [ 7 | "number", 8 | "string" 9 | ] 10 | }, 11 | "maximum": { 12 | "title": "Maximum value", 13 | "type": [ 14 | "number", 15 | "string" 16 | ] 17 | }, 18 | "setValue": { 19 | "title": "Set of values", 20 | "type": "array", 21 | "minItems": 1, 22 | "items": { 23 | "description": "For each field only the original data type of the property can occur (except for arrays), but we can't validate that in JSON Schema yet. See the sumamry description in the STAC specification for details." 24 | } 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /formats/protobuf/schema_test.go: -------------------------------------------------------------------------------- 1 | package protobuf_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/raystack/stencil/core/schema" 7 | "github.com/raystack/stencil/formats/protobuf" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func getParsedSchema(t *testing.T) schema.ParsedSchema { 12 | t.Helper() 13 | data := getDescriptorData(t, "./testdata/valid", true) 14 | sc, err := protobuf.GetParsedSchema(data) 15 | assert.NoError(t, err) 16 | return sc 17 | } 18 | 19 | func TestParsedSchema(t *testing.T) { 20 | t.Run("getCanonicalValue", func(t *testing.T) { 21 | sc := getParsedSchema(t) 22 | scFile := sc.GetCanonicalValue() 23 | assert.ElementsMatch(t, scFile.Fields, []string{"google.protobuf.Duration.seconds", 24 | "google.protobuf.Duration.nanos", 25 | "a.Test.field1", 26 | "a.Test.field2"}) 27 | assert.ElementsMatch(t, []string{"google.protobuf.Duration", "a.Test"}, scFile.Types) 28 | }) 29 | } 30 | -------------------------------------------------------------------------------- /docs/docs/usecases.md: -------------------------------------------------------------------------------- 1 | # Usecases 2 | 3 | This page describes popular Stencil use cases and provides related resources that you can use to create Stencil workflows. 4 | 5 | ## Event-driven architecture 6 | 7 | Event-driven architecture is a software paradigm that promotes using events as a means of communication between decoupled services. Events are the records of a change in state, e.g. a customer booking an order, driver, a driver confirming the booking, etc. Events are immutable and are usually ordered in the sequence of their creation. 8 | 9 | Event-driven architecture usually has three key components: producers, message brokers, and consumers. Consumer and producer services are loosely where event producers don't know which events consumers are listening to. This decoupling allows producers and consumers to evolve, scale, and deploy independently. 10 | 11 | But this also opens a challenge for managing data schema across consumers and producers. 12 | -------------------------------------------------------------------------------- /core/schema/provider/provider.go: -------------------------------------------------------------------------------- 1 | package provider 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/raystack/stencil/core/schema" 7 | "github.com/raystack/stencil/formats/avro" 8 | "github.com/raystack/stencil/formats/json" 9 | "github.com/raystack/stencil/formats/protobuf" 10 | ) 11 | 12 | type parseFn func([]byte) (schema.ParsedSchema, error) 13 | 14 | type SchemaProvider struct { 15 | mapper map[string]parseFn 16 | } 17 | 18 | func (s *SchemaProvider) ParseSchema(format string, data []byte) (schema.ParsedSchema, error) { 19 | fn, ok := s.mapper[format] 20 | if ok { 21 | return fn(data) 22 | } 23 | return nil, errors.New("unknown schema") 24 | } 25 | 26 | func NewSchemaProvider() *SchemaProvider { 27 | mp := make(map[string]parseFn) 28 | mp["FORMAT_PROTOBUF"] = protobuf.GetParsedSchema 29 | mp["FORMAT_AVRO"] = avro.ParseSchema 30 | mp["FORMAT_JSON"] = json.GetParsedSchema 31 | return &SchemaProvider{ 32 | mapper: mp, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /.github/workflows/test-clojure-client.yml: -------------------------------------------------------------------------------- 1 | name: Test stencil clients 2 | on: 3 | push: 4 | paths: 5 | - "clients/clojure/**" 6 | branches: 7 | - main 8 | pull_request: 9 | paths: 10 | - "clients/clojure/**" 11 | branches: 12 | - main 13 | jobs: 14 | test-clojure-client: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - uses: actions/checkout@v2 18 | - name: Set up JDK 8 19 | uses: actions/setup-java@v2 20 | with: 21 | distribution: adopt 22 | java-version: 8 23 | - name: Install clojure tools 24 | uses: DeLaGuardo/setup-clojure@4.0 25 | with: 26 | lein: 2.9.8 27 | github-token: ${{ secrets.GITHUB_TOKEN }} 28 | - name: check formatting 29 | run: lein cljfmt check 30 | working-directory: clients/clojure 31 | - name: Run tests 32 | run: lein test 33 | working-directory: clients/clojure 34 | -------------------------------------------------------------------------------- /.github/workflows/docs.yml: -------------------------------------------------------------------------------- 1 | name: docs 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | workflow_dispatch: 8 | 9 | jobs: 10 | documentation: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v2 14 | - uses: actions/setup-node@v2 15 | - name: Installation 16 | uses: bahmutov/npm-install@v1 17 | with: 18 | install-command: yarn 19 | working-directory: docs 20 | - name: Build docs 21 | working-directory: docs 22 | run: cd docs && yarn build 23 | - name: Deploy docs 24 | env: 25 | GIT_USER: ravisuhag 26 | GIT_PASS: ${{ secrets.DOCU_RS_TOKEN }} 27 | DEPLOYMENT_BRANCH: gh-pages 28 | CURRENT_BRANCH: main 29 | working-directory: docs 30 | run: | 31 | git config --global user.email "suhag.ravi@gmail.com" 32 | git config --global user.name "ravisuhag" 33 | yarn deploy 34 | -------------------------------------------------------------------------------- /formats/json/testdata/typeChecks/typeCheckSchema.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "summaries", 3 | "properties": { 4 | "objectType": { 5 | "title": "Minimum value", 6 | "type": "object"}, 7 | "arrayType": { 8 | "title": "Maximum value", 9 | "type": "array", 10 | "items": { 11 | "description": "For each field only the original data type of the property can occur (except for arrays), but we can't validate that in JSON Schema yet. See the sumamry description in the STAC specification for details.", 12 | "type":"integer" 13 | } 14 | }, 15 | "integerType": { 16 | "title": "Set of values", 17 | "type": "integer", 18 | "minItems": 1 19 | }, 20 | "stringType": { 21 | "type": "string" 22 | }, 23 | "numberType": { 24 | "type": "number" 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /config/config.yaml: -------------------------------------------------------------------------------- 1 | # Raystack Stencil Configuration 2 | # 3 | # 4 | # !!WARNING!! 5 | # This configuration file is for documentation purposes only. Do not use it in production. 6 | # 7 | # Stencil can be configured using a configuration file and passing the file location using `--config path/to/config.yaml`. 8 | # Per default, Stencil will look up and load file ~/.stencil.yaml. All configuration keys can be set using environment 9 | # variables as well. 10 | # 11 | 12 | # server controls the configuration for the GRPC server. 13 | # The port to listen on. Defaults to 8080 14 | port: 8080 15 | # Timeout represents graceful shutdown period. Defaults to 60 sec 16 | timeout: 5s 17 | # Configuration for profiling application with new relic 18 | newrelic: 19 | appname: example 20 | enabled: false 21 | license: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" 22 | # Database configurations for stencil backend 23 | db: 24 | # Connection string for postgres database 25 | connectionstring: "postgres://postgres@localhost:5432/db" 26 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Desktop (please complete the following information):** 27 | - OS: [e.g. iOS] 28 | - Browser [e.g. chrome, safari] 29 | - Version [e.g. 22] 30 | 31 | **Smartphone (please complete the following information):** 32 | - Device: [e.g. iPhone6] 33 | - OS: [e.g. iOS8.1] 34 | - Browser [e.g. stock browser, safari] 35 | - Version [e.g. 22] 36 | 37 | **Additional context** 38 | Add any other context about the problem here. 39 | -------------------------------------------------------------------------------- /clients/python/test/test_client.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import patch 2 | 3 | from google.protobuf.reflection import GeneratedProtocolMessageType 4 | 5 | from src.raystack.stencil import Client 6 | from src.raystack.store import Store 7 | 8 | URL = 'http://stencil.test/proto-descriptors/test/latest' 9 | 10 | 11 | def get_file_desc(): 12 | with open('data/one.desc', 'rb') as myfile: 13 | desc = myfile.read() 14 | return desc 15 | 16 | 17 | def test_store(protoc_setup): 18 | file_desc = get_file_desc() 19 | store = Store() 20 | store.load(data=file_desc) 21 | assert 'test.One' in store.data 22 | assert isinstance(store.get('test.One'), store.get('test.One').__class__) 23 | 24 | 25 | @patch('raystack.store.Store._load_from_url') 26 | def test_client(test_desc_from_url, protoc_setup): 27 | file_desc = get_file_desc() 28 | test_desc_from_url.return_value = file_desc 29 | 30 | client = Client(URL) 31 | 32 | assert isinstance(client.get_descriptor('test.One'), GeneratedProtocolMessageType) 33 | -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: "Lint" 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | golangci: 7 | name: "Lint" 8 | runs-on: "ubuntu-latest" 9 | steps: 10 | - uses: actions/setup-go@v5 11 | - uses: actions/checkout@v4 12 | - name: Crete empty build directory 13 | run: mkdir ui/build && touch ui/build/.gitkeep 14 | - name: golangci-lint 15 | uses: golangci/golangci-lint-action@v6 16 | with: 17 | version: v1.64 18 | 19 | codeql: 20 | name: "Analyze with CodeQL" 21 | runs-on: "ubuntu-latest" 22 | permissions: 23 | actions: "read" 24 | contents: "read" 25 | security-events: "write" 26 | strategy: 27 | fail-fast: false 28 | matrix: 29 | language: ["go"] 30 | steps: 31 | - uses: "actions/checkout@v2" 32 | - uses: "github/codeql-action/init@v1" 33 | with: 34 | languages: "${{ matrix.language }}" 35 | - uses: "github/codeql-action/autobuild@v1" 36 | - uses: "github/codeql-action/analyze@v1" 37 | -------------------------------------------------------------------------------- /formats/json/error.go: -------------------------------------------------------------------------------- 1 | package json 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | type diffKind int 9 | 10 | type diff struct { 11 | kind diffKind 12 | msg string 13 | } 14 | 15 | type compatibilityErr struct { 16 | notAllowed []diffKind 17 | diffs []diff 18 | } 19 | 20 | func (d diffKind) contains(others []diffKind) bool { 21 | for _, v := range others { 22 | if v == d { 23 | return true 24 | } 25 | } 26 | return false 27 | } 28 | 29 | func (c *compatibilityErr) add(kind diffKind, location string, format string, args ...interface{}) { 30 | msg := fmt.Sprintf(format, args...) 31 | if kind.contains(c.notAllowed) && msg != "" { 32 | c.diffs = append(c.diffs, diff{kind: kind, msg: fmt.Sprintf("%s: %s", location, msg)}) 33 | } 34 | } 35 | 36 | func (c *compatibilityErr) isEmpty() bool { 37 | return len(c.diffs) == 0 38 | } 39 | 40 | func (c *compatibilityErr) Error() string { 41 | var msgs []string 42 | for _, val := range c.diffs { 43 | msgs = append(msgs, val.msg) 44 | } 45 | return strings.Join(msgs, ";") 46 | } 47 | -------------------------------------------------------------------------------- /internal/server/graceful.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "log" 8 | "net/http" 9 | "os" 10 | "os/signal" 11 | "syscall" 12 | 13 | "github.com/raystack/stencil/config" 14 | ) 15 | 16 | func runWithGracefulShutdown(config *config.Config, router http.Handler, cleanUp func()) { 17 | srv := &http.Server{ 18 | Addr: fmt.Sprintf(":%s", config.Port), 19 | Handler: router, 20 | } 21 | go func() { 22 | if err := srv.ListenAndServe(); err != nil && errors.Is(err, http.ErrServerClosed) { 23 | log.Printf("listen: %s\n", err) 24 | } 25 | }() 26 | 27 | quit := make(chan os.Signal, 2) 28 | 29 | signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) 30 | <-quit 31 | log.Println("Shutting down server...") 32 | 33 | ctx, cancel := context.WithTimeout(context.Background(), config.Timeout) 34 | defer cancel() 35 | 36 | if err := srv.Shutdown(ctx); err != nil { 37 | cleanUp() 38 | log.Fatal("Server forced to shutdown:", err) 39 | } 40 | cleanUp() 41 | 42 | log.Println("Server exiting") 43 | } 44 | -------------------------------------------------------------------------------- /clients/python/src/raystack/stencil.py: -------------------------------------------------------------------------------- 1 | from schedule import Scheduler 2 | from google.protobuf.message import Message 3 | 4 | from raystack.store import Store 5 | 6 | 7 | class MultiUrlClient: 8 | def __init__(self, urls:list, interval=3600, auto_refresh=False) -> None: 9 | self._store = Store() 10 | self._urls = urls 11 | self._interval = interval 12 | self._auto_refresh = auto_refresh 13 | self._scheduler = Scheduler() 14 | if self._auto_refresh: 15 | self._scheduler.every(self._interval).seconds.do(self.refresh) 16 | self.refresh() 17 | 18 | def refresh(self): 19 | for url in self._urls: 20 | self._store.load(url=url) 21 | 22 | def get_descriptor(self, name: str) -> Message: 23 | return self._store.get(name) 24 | 25 | def parse(self, name: str, data: bytes): 26 | msg = self.get_descriptor(name) 27 | return msg.ParseFromString(data) 28 | 29 | 30 | class Client(MultiUrlClient): 31 | def __init__(self, url: str) -> None: 32 | super().__init__([url]) 33 | -------------------------------------------------------------------------------- /.github/workflows/release-server.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | on: 3 | push: 4 | tags: 5 | - "v*.*.*" 6 | workflow_dispatch: 7 | inputs: 8 | goreleaserArgs: 9 | required: false 10 | type: string 11 | 12 | jobs: 13 | publish-server: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - name: Checkout 17 | uses: actions/checkout@v3 18 | with: 19 | fetch-depth: 0 20 | - name: Set up Go 21 | uses: actions/setup-go@v4 22 | with: 23 | go-version: "1.20" 24 | - name: Login to DockerHub 25 | uses: docker/login-action@v1 26 | with: 27 | registry: docker.io 28 | username: ${{ secrets.DOCKERHUB_USERNAME }} 29 | password: ${{ secrets.DOCKERHUB_TOKEN }} 30 | - name: Run GoReleaser 31 | uses: goreleaser/goreleaser-action@v5 32 | with: 33 | distribution: goreleaser 34 | version: v1.21.2 35 | args: --rm-dist ${{ inputs.goreleaserArgs }} 36 | env: 37 | GITHUB_TOKEN: ${{ secrets.GO_RELEASER_TOKEN }} 38 | -------------------------------------------------------------------------------- /clients/python/setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | with open("README.md", "r", encoding="utf-8") as fh: 4 | long_description = fh.read() 5 | 6 | setuptools.setup( 7 | name="stencil-python-client", 8 | version="0.0.1", 9 | author="Raystack", 10 | author_email="raystack@gmail.com", 11 | description="Stencil Python client package provides a store to lookup protobuf descriptors and options to keep the protobuf descriptors upto date.", 12 | long_description=long_description, 13 | long_description_content_type="text/markdown", 14 | url="https://github.com/raystack/stencil", 15 | project_urls={ 16 | "Bug Tracker": "https://github.com/raystack/stencil/issues", 17 | }, 18 | classifiers=[ 19 | "Programming Language :: Python :: 3", 20 | "License :: OSI Approved :: Apache Software License", 21 | "Operating System :: OS Independent", 22 | ], 23 | package_dir={"": "src"}, 24 | packages=setuptools.find_packages(where="src"), 25 | python_requires=">=3.6", 26 | install_requires=[ 27 | 'protobuf', 28 | 'schedule', 29 | 'requests' 30 | ] 31 | ) 32 | -------------------------------------------------------------------------------- /.github/workflows/release-java-client.yml: -------------------------------------------------------------------------------- 1 | name: Release Stencil Java Client 2 | on: 3 | push: 4 | tags: 5 | - "v*.*.*" 6 | workflow_dispatch: 7 | 8 | jobs: 9 | publish-java-client: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v2 13 | - name: Set up JDK 8 14 | uses: actions/setup-java@v2 15 | with: 16 | distribution: adopt 17 | java-version: 8 18 | - name: Publish java client 19 | run: | 20 | printf "$GPG_SIGNING_KEY" | base64 --decode > private.key 21 | ./gradlew clean publishToSonatype closeAndReleaseSonatypeStagingRepository -Psigning.keyId=${GPG_SIGNING_KEY_ID} -Psigning.password=${GPG_SIGNING_PASSWORD} -Psigning.secretKeyRingFile=private.key --console=verbose 22 | working-directory: clients/java 23 | env: 24 | MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }} 25 | MAVEN_PASSWORD: ${{ secrets.OSSRH_TOKEN }} 26 | GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} 27 | GPG_SIGNING_KEY_ID: ${{ secrets.GPG_SIGNING_KEY_ID }} 28 | GPG_SIGNING_PASSWORD: ${{ secrets.GPG_SIGNING_PASSWORD }} 29 | -------------------------------------------------------------------------------- /formats/protobuf/error.go: -------------------------------------------------------------------------------- 1 | package protobuf 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "google.golang.org/grpc/codes" 8 | "google.golang.org/grpc/status" 9 | "google.golang.org/protobuf/reflect/protoreflect" 10 | ) 11 | 12 | type diff struct { 13 | kind diffKind 14 | msg string 15 | } 16 | 17 | type compatibilityErr struct { 18 | notAllowed []diffKind 19 | diffs []diff 20 | } 21 | 22 | func (c *compatibilityErr) add(kind diffKind, desc protoreflect.Descriptor, format string, args ...interface{}) { 23 | msg := fmt.Sprintf(format, args...) 24 | path := desc.ParentFile().Path() 25 | if kind.contains(c.notAllowed) && msg != "" { 26 | c.diffs = append(c.diffs, diff{kind: kind, msg: fmt.Sprintf("%s: %s", path, msg)}) 27 | } 28 | } 29 | 30 | func (c *compatibilityErr) isEmpty() bool { 31 | return len(c.diffs) == 0 32 | } 33 | 34 | func (c *compatibilityErr) Error() string { 35 | var msgs []string 36 | for _, val := range c.diffs { 37 | msgs = append(msgs, val.msg) 38 | } 39 | return strings.Join(msgs, ";") 40 | } 41 | 42 | func (c *compatibilityErr) GRPCStatus() *status.Status { 43 | return status.New(codes.InvalidArgument, c.Error()) 44 | } 45 | -------------------------------------------------------------------------------- /formats/protobuf/testdata/backward/current/1.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package a; 4 | 5 | enum TestEnum { 6 | UNSPECIFIED = 0; 7 | ONE = 1; 8 | } 9 | 10 | enum NewTestEnum { 11 | NEW_UNSPECIFIED = 0; 12 | NEW_ONE = 1; 13 | } 14 | // Unchange message 15 | message One { 16 | string field_one = 1; 17 | } 18 | 19 | // Backward compatible message 20 | message Two { 21 | reserved 4,5; 22 | reserved "deleted_field"; 23 | string string_field = 1; 24 | int64 int_field = 2; 25 | TestEnum enum_field = 3; 26 | string new_field = 6; 27 | enum CompatibleEnum { 28 | UNKNOWN = 0; 29 | ONE = 1; 30 | TWO = 2; 31 | THREE = 3; 32 | } 33 | } 34 | 35 | message NewMessage { 36 | string one = 1; 37 | } 38 | 39 | message BreakingMessage { 40 | string name_changed = 1; 41 | string number_change = 11; 42 | int64 num_exchange_a = 4; 43 | string num_exchange_b = 3; 44 | int64 kind_change = 5; 45 | NewTestEnum type_name_change = 6; 46 | NewMessage type_message_change = 7; 47 | string cardinality_field = 8; 48 | enum BreackingEnum { 49 | reserved 4, 5, 11 to 14; 50 | UNKNOWN = 0; 51 | NUMBER_CHANGE = 2; 52 | NAME_CHANGED = 3; 53 | } 54 | } 55 | 56 | 57 | -------------------------------------------------------------------------------- /core/search/service.go: -------------------------------------------------------------------------------- 1 | package search 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | ) 7 | 8 | var ( 9 | ErrEmptyQueryString = errors.New("query string cannot be empty") 10 | ErrEmptySchemaID = errors.New("schema_id cannot be empty") 11 | ErrEmptyNamespaceID = errors.New("namespace_id cannot be empty") 12 | ) 13 | 14 | type Service struct { 15 | repo Repository 16 | } 17 | 18 | func NewService(repository Repository) *Service { 19 | return &Service{ 20 | repo: repository, 21 | } 22 | } 23 | 24 | func (s *Service) Search(ctx context.Context, req *SearchRequest) (*SearchResponse, error) { 25 | if req.Query == "" { 26 | return nil, ErrEmptyQueryString 27 | } 28 | 29 | if req.SchemaID != "" && req.NamespaceID == "" { 30 | return nil, ErrEmptyNamespaceID 31 | } 32 | 33 | var res []*SearchHits 34 | var err error 35 | if req.VersionID == 0 && !req.History { 36 | res, err = s.repo.SearchLatest(ctx, req) 37 | } else { 38 | if req.VersionID > 0 && req.SchemaID == "" { 39 | return nil, ErrEmptySchemaID 40 | } 41 | res, err = s.repo.Search(ctx, req) 42 | } 43 | 44 | if err != nil { 45 | return nil, err 46 | } 47 | return &SearchResponse{ 48 | Hits: res, 49 | }, nil 50 | } 51 | -------------------------------------------------------------------------------- /docs/src/core/Container.js: -------------------------------------------------------------------------------- 1 | import classNames from 'classnames'; 2 | import * as React from 'react'; 3 | 4 | const Container = props => { 5 | const containerClasses = classNames(props.className, { 6 | darkBackground: props.background === 'dark', 7 | highlightBackground: props.background === 'highlight', 8 | lightBackground: props.background === 'light', 9 | paddingAll: props.padding.indexOf('all') >= 0, 10 | paddingBottom: props.padding.indexOf('bottom') >= 0, 11 | paddingLeft: props.padding.indexOf('left') >= 0, 12 | paddingRight: props.padding.indexOf('right') >= 0, 13 | paddingTop: props.padding.indexOf('top') >= 0, 14 | }); 15 | let wrappedChildren; 16 | 17 | if (props.wrapper) { 18 | wrappedChildren =
{props.children}
; 19 | } else { 20 | wrappedChildren = props.children; 21 | } 22 | return ( 23 |
24 | {wrappedChildren} 25 |
26 | ); 27 | }; 28 | 29 | Container.defaultProps = { 30 | background: null, 31 | padding: [], 32 | wrapper: true, 33 | }; 34 | 35 | export default Container; 36 | -------------------------------------------------------------------------------- /ui/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ui", 3 | "version": "0.1.0", 4 | "private": true, 5 | "homepage": "/ui", 6 | "dependencies": { 7 | "@testing-library/jest-dom": "^5.16.4", 8 | "@testing-library/react": "^13.1.1", 9 | "@testing-library/user-event": "^13.5.0", 10 | "@types/jest": "^27.4.1", 11 | "@types/node": "^16.11.31", 12 | "@types/react": "^18.0.8", 13 | "@types/react-dom": "^18.0.0", 14 | "react": "^18.1.0", 15 | "react-dom": "^18.1.0", 16 | "react-scripts": "5.0.1", 17 | "styled-components": "^5.2.1", 18 | "typescript": "^4.6.3", 19 | "web-vitals": "^2.1.4" 20 | }, 21 | "scripts": { 22 | "start": "react-scripts start", 23 | "build": "react-scripts build", 24 | "test": "react-scripts test", 25 | "eject": "react-scripts eject" 26 | }, 27 | "eslintConfig": { 28 | "extends": [ 29 | "react-app", 30 | "react-app/jest" 31 | ] 32 | }, 33 | "browserslist": { 34 | "production": [ 35 | ">0.2%", 36 | "not dead", 37 | "not op_mini all" 38 | ], 39 | "development": [ 40 | "last 1 chrome version", 41 | "last 1 firefox version", 42 | "last 1 safari version" 43 | ] 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /clients/README.md: -------------------------------------------------------------------------------- 1 | # Stencil Clients 2 | 3 | Stencil clients abstracts handling of descriptorset file on client side. Currently we officially support Stencil client in Java, Go, JS languages. 4 | 5 | ## Features 6 | 7 | - downloading of descriptorset file from server 8 | - parse API to deserialize protobuf encoded messages 9 | - lookup API to find proto descriptors 10 | - inbuilt strategies to refresh protobuf schema definitions. 11 | 12 | ## A note on configuring Stencil clients 13 | 14 | - Stencil server provides API to download latest descriptor file. If new version is available latest file will point to new descriptor file. Always use latest version proto descriptor url for stencil client if you want to refresh schema definitions in runtime. 15 | - Keep the refresh intervals relatively large (eg: 24hrs or 12 hrs) to reduce the number of calls depending on how fast systems produce new messages using new proto schema. 16 | - You can refresh descriptor file only if unknowns fields are faced by the client while parsing. This reduces unneccessary frequent calls made by clients. Currently this feature supported in JAVA and GO clients. 17 | 18 | ## Languages 19 | 20 | - [Java](java) 21 | - [Go](go) 22 | - [Javascript](js) 23 | -------------------------------------------------------------------------------- /formats/json/provider_test.go: -------------------------------------------------------------------------------- 1 | package json_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/raystack/stencil/formats/json" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestSchemaValidation(t *testing.T) { 11 | for _, test := range []struct { 12 | name string 13 | schema string 14 | isError bool 15 | }{ 16 | {"should return error if schema is not json", `{invalid_json,}`, true}, 17 | {"should return error if schema is not valid json", `{ 18 | "$id": "https://example.com/address.schema.json", 19 | "type": "object", 20 | "properties": { 21 | "f1": { 22 | "type": "string" 23 | } 24 | }, 25 | "required": "this is supposed to array" 26 | }`, true}, 27 | {"should return nil if schema is valid json", `{ 28 | "$id": "https://example.com/address.schema.json", 29 | "type": "object", 30 | "properties": { 31 | "f1": { 32 | "type": "string" 33 | } 34 | }, 35 | "required": ["f1"] 36 | }`, false}, 37 | } { 38 | t.Run(test.name, func(t *testing.T) { 39 | _, err := json.GetParsedSchema([]byte(test.schema)) 40 | if test.isError { 41 | assert.Error(t, err) 42 | } else { 43 | assert.Nil(t, err) 44 | } 45 | }) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /clients/js/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@raystack/stencil", 3 | "version": "0.5.1", 4 | "description": "Stencil js client package provides a store to lookup protobuf descriptors and options to keep the protobuf descriptors upto date.", 5 | "main": "main.js", 6 | "scripts": { 7 | "test": "jest", 8 | "lint": "eslint .", 9 | "format": "prettier --check '**/*.js'", 10 | "format:fix": "prettier '**/*.js' --write" 11 | }, 12 | "keywords": [ 13 | "stencil", 14 | "protobuf-schema-registry" 15 | ], 16 | "author": "", 17 | "license": "Apache-2.0", 18 | "dependencies": { 19 | "node-fetch": "^2.6.7", 20 | "protobufjs": "^6.10.2" 21 | }, 22 | "devDependencies": { 23 | "eslint": "^7.25.0", 24 | "eslint-config-airbnb-base": "^14.2.1", 25 | "eslint-config-prettier": "^8.3.0", 26 | "eslint-config-standard": "^16.0.2", 27 | "eslint-plugin-import": "^2.22.1", 28 | "eslint-plugin-node": "^11.1.0", 29 | "eslint-plugin-promise": "^4.3.1", 30 | "jest": "^26.6.3", 31 | "prettier": "2.2.1" 32 | }, 33 | "prettier": { 34 | "trailingComma": "none", 35 | "tabWidth": 2, 36 | "semi": true, 37 | "singleQuote": true, 38 | "printWidth": 80 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /.github/workflows/test-server.yaml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | workflow_dispatch: 9 | 10 | jobs: 11 | test: 12 | runs-on: ubuntu-latest 13 | services: 14 | postgres: 15 | image: postgres:13 16 | env: 17 | POSTGRES_HOST: localhost 18 | POSTGRES_USER: postgres 19 | POSTGRES_PASSWORD: postgres 20 | POSTGRES_DB: test_stencil_db 21 | options: >- 22 | --health-cmd pg_isready 23 | --health-interval 10s 24 | --health-timeout 5s 25 | --health-retries 5 26 | ports: 27 | - 5432:5432 28 | steps: 29 | - name: Set up Go 1.x 30 | uses: actions/setup-go@v4 31 | with: 32 | go-version: ^1.20 33 | id: go 34 | - name: Install Protoc 35 | uses: arduino/setup-protoc@v1 36 | with: 37 | version: "3.x" 38 | - name: Check out code into the Go module directory 39 | uses: actions/checkout@v2 40 | - name: Test 41 | run: make test 42 | env: 43 | TEST_DB_CONNECTIONSTRING: "postgres://postgres:postgres@localhost:5432/test_stencil_db?sslmode=disable" 44 | -------------------------------------------------------------------------------- /clients/java/src/main/java/io/odpf/stencil/client/StencilClient.java: -------------------------------------------------------------------------------- 1 | package org.raystack.stencil.client; 2 | 3 | import com.google.protobuf.Descriptors; 4 | import com.google.protobuf.DynamicMessage; 5 | import com.google.protobuf.InvalidProtocolBufferException; 6 | import org.raystack.stencil.Parser; 7 | import org.raystack.stencil.exception.StencilRuntimeException; 8 | 9 | import java.io.Closeable; 10 | import java.util.Map; 11 | 12 | /** 13 | * A client to get the protobuf descriptors and more information 14 | */ 15 | public interface StencilClient extends Closeable { 16 | Descriptors.Descriptor get(String className); 17 | 18 | default DynamicMessage parse(String className, byte[] data) throws InvalidProtocolBufferException { 19 | Descriptors.Descriptor descriptor = get(className); 20 | if (descriptor == null) { 21 | throw new StencilRuntimeException(new Throwable(String.format("No Descriptors found for %s", className))); 22 | } 23 | return DynamicMessage.parseFrom(descriptor, data); 24 | } 25 | 26 | default Parser getParser(String className) { 27 | return (data) -> parse(className, data); 28 | } 29 | 30 | Map getAll(); 31 | 32 | void refresh(); 33 | } 34 | -------------------------------------------------------------------------------- /docs/docs/guides/0_introduction.md: -------------------------------------------------------------------------------- 1 | import Tabs from "@theme/Tabs"; 2 | import TabItem from "@theme/TabItem"; 3 | 4 | # Introduction 5 | 6 | This tour introduces you to Stencil schema registry. Along the way you will learn how to manage schemas, enforce rules, serialise and deserialise data using stencil clients. 7 | 8 | ### Prerequisites 9 | 10 | This tour requires you to have Stencil CLI tool installed on your local machine. You can run `stencil version` to verify the installation. Please follow installation guide if you do not have it installed already. 11 | 12 | Stencil CLI and clients talks to Stencil server to publish and fetch schema. Please make sure you also have a stencil server running. You can also run server locally with `stencil server start` command. For more details check deployment guide. 13 | 14 | ### Help 15 | 16 | At any time you can run the following commands. 17 | 18 | ```bash 19 | # Check the installed version for stencil cli tool 20 | $ stencil version 21 | 22 | # See the help for a command 23 | $ stencil --help 24 | ``` 25 | 26 | Help command can also be run on any sub command. 27 | 28 | ```bash 29 | $ stencil schema --help 30 | ``` 31 | 32 | Check the reference for stencil cli commands. 33 | 34 | ```bash 35 | $ stencil reference 36 | ``` 37 | -------------------------------------------------------------------------------- /formats/protobuf/testdata/backward/previous/1.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package a; 4 | 5 | enum TestEnum { 6 | UNSPECIFIED = 0; 7 | ONE = 1; 8 | } 9 | // Unchange message 10 | message One { 11 | string field_one = 1; 12 | } 13 | // This message will be deleted 14 | message WillBeDeleted { 15 | string one = 1; 16 | int64 two = 2; 17 | } 18 | // This enum will be deleted 19 | enum EnumWillBeDeleted { 20 | UNKNOWN = 0; 21 | TWO = 2; 22 | } 23 | // Backward compatible message 24 | // field addition, enum addition, message addition 25 | message Two { 26 | reserved 4,5; 27 | reserved "deleted_field"; 28 | string string_field = 1; 29 | int64 int_field = 2; 30 | TestEnum enum_field = 3; 31 | enum CompatibleEnum { 32 | UNKNOWN = 0; 33 | ONE = 1; 34 | TWO = 2; 35 | } 36 | } 37 | 38 | message BreakingMessage { 39 | string name_change = 1; 40 | string number_change = 2; 41 | int64 num_exchange_a = 3; 42 | string num_exchange_b = 4; 43 | string kind_change = 5; 44 | TestEnum type_name_change = 6; 45 | One type_message_change = 7; 46 | repeated string cardinality_field = 8; 47 | enum BreackingEnum { 48 | reserved 4, 5, 8, 11 to 15; 49 | reserved "never_existed"; 50 | UNKNOWN = 0; 51 | NUMBER_CHANGE = 1; 52 | NAME_CHANGE = 3; 53 | } 54 | } 55 | 56 | 57 | -------------------------------------------------------------------------------- /docs/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "stencil", 3 | "version": "0.0.0", 4 | "private": true, 5 | "scripts": { 6 | "docusaurus": "docusaurus", 7 | "start": "docusaurus start", 8 | "build": "docusaurus build", 9 | "swizzle": "docusaurus swizzle", 10 | "deploy": "docusaurus deploy", 11 | "clear": "docusaurus clear", 12 | "serve": "docusaurus serve", 13 | "write-translations": "docusaurus write-translations", 14 | "write-heading-ids": "docusaurus write-heading-ids" 15 | }, 16 | "dependencies": { 17 | "@docusaurus/core": "^2.0.0-beta.17", 18 | "@docusaurus/plugin-google-gtag": "^2.0.0-beta.17", 19 | "@docusaurus/preset-classic": "^2.0.0-beta.17", 20 | "@mdx-js/react": "^1.6.21", 21 | "@svgr/webpack": "^6.0.0", 22 | "classnames": "^2.3.1", 23 | "clsx": "^1.1.1", 24 | "file-loader": "^6.2.0", 25 | "prism-react-renderer": "^1.2.1", 26 | "react": "^17.0.1", 27 | "react-dom": "^17.0.1", 28 | "url-loader": "^4.1.1" 29 | }, 30 | "browserslist": { 31 | "production": [ 32 | ">0.5%", 33 | "not dead", 34 | "not op_mini all" 35 | ], 36 | "development": [ 37 | "last 1 chrome version", 38 | "last 1 firefox version", 39 | "last 1 safari version" 40 | ] 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /docs/docs/clients/overview.md: -------------------------------------------------------------------------------- 1 | # Overview 2 | 3 | Stencil clients abstracts handling of descriptorset file on client side. Currently we officially support Stencil client in Java, Go, JS languages. 4 | 5 | ## Features 6 | 7 | - downloading of descriptorset file from server 8 | - parse API to deserialize protobuf encoded messages 9 | - lookup API to find proto descriptors 10 | - inbuilt strategies to refresh protobuf schema definitions. 11 | 12 | ## A note on configuring Stencil clients 13 | 14 | - Stencil server provides API to download latest descriptor file. If new version is available latest file will point to new descriptor file. Always use latest version proto descriptor url for stencil client if you want to refresh schema definitions in runtime. 15 | - Keep the refresh intervals relatively large (eg: 24hrs or 12 hrs) to reduce the number of calls depending on how fast systems produce new messages using new proto schema. 16 | - You can refresh descriptor file only if unknowns fields are faced by the client while parsing. This reduces unneccessary frequent calls made by clients. Currently this feature supported in JAVA and GO clients. 17 | 18 | ## Languages 19 | 20 | - [Java](java) 21 | - [Go](go) 22 | - [Javascript](js) 23 | - [Clojure](clojure) 24 | - Ruby - Coming soon 25 | - Python - Coming soon 26 | -------------------------------------------------------------------------------- /clients/go/protoutils.go: -------------------------------------------------------------------------------- 1 | package stencil 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "google.golang.org/protobuf/proto" 8 | "google.golang.org/protobuf/reflect/protodesc" 9 | "google.golang.org/protobuf/reflect/protoreflect" 10 | "google.golang.org/protobuf/reflect/protoregistry" 11 | "google.golang.org/protobuf/types/descriptorpb" 12 | ) 13 | 14 | func getJavaPackage(fileDesc protoreflect.FileDescriptor) string { 15 | file := protodesc.ToFileDescriptorProto(fileDesc) 16 | options := file.Options 17 | if options != nil && options.JavaPackage != nil { 18 | return *options.JavaPackage 19 | } 20 | return "" 21 | } 22 | 23 | func defaultKeyFn(msg protoreflect.MessageDescriptor) string { 24 | fullName := string(msg.FullName()) 25 | file := msg.ParentFile() 26 | protoPackage := string(file.Package()) 27 | pkg := getJavaPackage(file) 28 | if pkg == "" { 29 | return fullName 30 | } else if protoPackage == "" { 31 | return fmt.Sprintf("%s.%s", pkg, fullName) 32 | } 33 | return strings.Replace(fullName, protoPackage, pkg, 1) 34 | } 35 | 36 | func getFilesRegistry(data []byte) (*protoregistry.Files, error) { 37 | msg := &descriptorpb.FileDescriptorSet{} 38 | err := proto.Unmarshal(data, msg) 39 | if err != nil { 40 | return nil, fmt.Errorf("invalid file descriptorset file. %w", err) 41 | } 42 | return protodesc.NewFiles(msg) 43 | } 44 | -------------------------------------------------------------------------------- /formats/protobuf/provider.go: -------------------------------------------------------------------------------- 1 | package protobuf 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/raystack/stencil/core/schema" 7 | "google.golang.org/protobuf/proto" 8 | "google.golang.org/protobuf/reflect/protodesc" 9 | "google.golang.org/protobuf/reflect/protoregistry" 10 | "google.golang.org/protobuf/types/descriptorpb" 11 | ) 12 | 13 | func getRegistry(fds *descriptorpb.FileDescriptorSet, data []byte) (*protoregistry.Files, error) { 14 | if err := proto.Unmarshal(data, fds); err != nil { 15 | return nil, fmt.Errorf("descriptor set file is not valid. %w", err) 16 | } 17 | files, err := protodesc.NewFiles(fds) 18 | if err != nil { 19 | return files, fmt.Errorf("file is not fully contained descriptor file. hint: generate file descriptorset with --include_imports option. %w", err) 20 | } 21 | return files, err 22 | } 23 | 24 | // GetParsedSchema converts data into enriched data type to deal with protobuf schema 25 | func GetParsedSchema(data []byte) (schema.ParsedSchema, error) { 26 | fds := &descriptorpb.FileDescriptorSet{} 27 | files, err := getRegistry(fds, data) 28 | if err != nil { 29 | return &Schema{ 30 | isValid: false, 31 | }, err 32 | } 33 | orderedData, _ := proto.MarshalOptions{Deterministic: true}.Marshal(fds) 34 | return &Schema{ 35 | isValid: err == nil, 36 | Files: files, 37 | data: orderedData, 38 | }, nil 39 | } 40 | -------------------------------------------------------------------------------- /formats/protobuf/testdata/forward/current/1.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package a; 4 | 5 | enum TestEnum { 6 | UNSPECIFIED = 0; 7 | ONE = 1; 8 | } 9 | 10 | enum NewTestEnum { 11 | NEW_UNSPECIFIED = 0; 12 | NEW_ONE = 1; 13 | } 14 | // Unchange message 15 | message One { 16 | string field_one = 1; 17 | } 18 | 19 | // Forward compatible message 20 | message Two { 21 | reserved 4,5,6; 22 | reserved "deleted_field", "going_to_delete"; 23 | string string_field = 1; 24 | int64 int_field = 2; 25 | TestEnum enum_field = 3; 26 | string new_field = 7; 27 | enum CompatibleEnum { 28 | reserved 2; 29 | reserved "TWO"; 30 | UNKNOWN = 0; 31 | ONE = 1; 32 | THREE = 3; 33 | } 34 | } 35 | 36 | message NewMessage { 37 | string one = 1; 38 | } 39 | 40 | message BreakingMessage { 41 | reserved "delete_without_reseve_num"; 42 | reserved 10, 2; 43 | string name_changed = 1; 44 | string number_change = 11; 45 | int64 num_exchange_a = 4; 46 | string num_exchange_b = 3; 47 | int64 kind_change = 5; 48 | NewTestEnum type_name_change = 6; 49 | NewMessage type_message_change = 7; 50 | enum BreackingEnum { 51 | reserved "DELETE_ENUM_WITHOUT_RESERVE_NUM"; 52 | reserved 6; 53 | UNKNOWN = 0; 54 | NUMBER_CHANGE = 2; 55 | NAME_CHANGED = 3; 56 | } 57 | } 58 | 59 | message ChangeReserve { 60 | reserved 1 to 3, 11; 61 | reserved "a", "c"; 62 | } 63 | 64 | 65 | -------------------------------------------------------------------------------- /clients/js/lib/refresh_strategy.js: -------------------------------------------------------------------------------- 1 | const fetch = require('node-fetch'); 2 | 3 | function checkStatus(res) { 4 | if (res.ok) { 5 | return res; 6 | } 7 | throw new Error('Unable to download descriptor file'); 8 | } 9 | 10 | const joinPath = (url, path) => { 11 | if (url.endsWith('/')) { 12 | return url + path; 13 | } 14 | return [url, path].join('/'); 15 | }; 16 | 17 | const longPollingRefresh = () => async (url, options) => 18 | fetch(url, options) 19 | .then(checkStatus) 20 | .then((res) => res.buffer()); 21 | 22 | const versionBasedRefresh = () => { 23 | let prevLatestVersion = 0; 24 | return async (url, options) => { 25 | const versionsURL = joinPath(url, 'versions'); 26 | const versions = await fetch(versionsURL, options) 27 | .then(checkStatus) 28 | .then((res) => res.json()) 29 | .then((data) => data.versions || []); 30 | const maxVersion = Math.max(...versions); 31 | if (!versions.length || maxVersion <= prevLatestVersion) { 32 | return null; 33 | } 34 | const versionedURL = joinPath(versionsURL, maxVersion.toString()); 35 | const buffer = await fetch(versionedURL, options) 36 | .then(checkStatus) 37 | .then((res) => res.buffer()); 38 | prevLatestVersion = maxVersion; 39 | return buffer; 40 | }; 41 | }; 42 | 43 | module.exports = { 44 | longPollingRefresh, 45 | versionBasedRefresh 46 | }; 47 | -------------------------------------------------------------------------------- /docs/sidebars.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | docsSidebar: [ 3 | 'introduction', 4 | 'usecases', 5 | 'installation', 6 | 'glossary', 7 | { 8 | type: "category", 9 | label: "Guides", 10 | items: [ 11 | "guides/introduction", 12 | "guides/quickstart", 13 | "guides/manage_namespace", 14 | "guides/manage_schemas", 15 | "guides/clients", 16 | ], 17 | }, 18 | { 19 | type: "category", 20 | label: "Formats", 21 | items: [ 22 | "formats/protobuf", 23 | "formats/avro", 24 | "formats/json", 25 | ], 26 | }, 27 | { 28 | type: "category", 29 | label: "Server", 30 | items: [ 31 | "server/overview", 32 | "server/rules", 33 | ], 34 | }, 35 | { 36 | type: "category", 37 | label: "Clients", 38 | items: [ 39 | "clients/overview", 40 | "clients/go", 41 | "clients/java", 42 | "clients/clojure", 43 | "clients/js", 44 | ], 45 | }, 46 | { 47 | type: "category", 48 | label: "Reference", 49 | items: [ 50 | "reference/api", 51 | "reference/cli", 52 | ], 53 | }, 54 | { 55 | type: "category", 56 | label: "Contribute", 57 | items: [ 58 | "contribute/contribution", 59 | ], 60 | }, 61 | 'roadmap', 62 | ], 63 | }; -------------------------------------------------------------------------------- /formats/json/testdata/additionalProperties/partialOpenContent.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "summaries", 3 | "properties": {}, 4 | "additionalProperties": { 5 | "anyOf": [ 6 | { 7 | "title": "JSON Schema", 8 | "type": "object", 9 | "minProperties": 1, 10 | "allOf": [ 11 | { 12 | "$ref": "http://json-schema.org/draft-07/schema" 13 | } 14 | ] 15 | }, 16 | { 17 | "title": "Range", 18 | "type": "object", 19 | "required": [ 20 | "minimum", 21 | "maximum" 22 | ], 23 | "properties": { 24 | "minimum": { 25 | "title": "Minimum value", 26 | "type": [ 27 | "number", 28 | "string" 29 | ] 30 | }, 31 | "maximum": { 32 | "title": "Maximum value", 33 | "type": [ 34 | "number", 35 | "string" 36 | ] 37 | } 38 | } 39 | }, 40 | { 41 | "title": "Set of values", 42 | "type": "array", 43 | "minItems": 1, 44 | "items": { 45 | "description": "For each field only the original data type of the property can occur (except for arrays), but we can't validate that in JSON Schema yet. See the sumamry description in the STAC specification for details." 46 | } 47 | } 48 | ] 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /core/schema/mocks/schema_provider.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v2.12.2. DO NOT EDIT. 2 | 3 | package mocks 4 | 5 | import ( 6 | schema "github.com/raystack/stencil/core/schema" 7 | mock "github.com/stretchr/testify/mock" 8 | 9 | testing "testing" 10 | ) 11 | 12 | // SchemaProvider is an autogenerated mock type for the Provider type 13 | type SchemaProvider struct { 14 | mock.Mock 15 | } 16 | 17 | // ParseSchema provides a mock function with given fields: format, data 18 | func (_m *SchemaProvider) ParseSchema(format string, data []byte) (schema.ParsedSchema, error) { 19 | ret := _m.Called(format, data) 20 | 21 | var r0 schema.ParsedSchema 22 | if rf, ok := ret.Get(0).(func(string, []byte) schema.ParsedSchema); ok { 23 | r0 = rf(format, data) 24 | } else { 25 | if ret.Get(0) != nil { 26 | r0 = ret.Get(0).(schema.ParsedSchema) 27 | } 28 | } 29 | 30 | var r1 error 31 | if rf, ok := ret.Get(1).(func(string, []byte) error); ok { 32 | r1 = rf(format, data) 33 | } else { 34 | r1 = ret.Error(1) 35 | } 36 | 37 | return r0, r1 38 | } 39 | 40 | // NewSchemaProvider creates a new instance of SchemaProvider. It also registers the testing.TB interface on the mock and a cleanup function to assert the mocks expectations. 41 | func NewSchemaProvider(t testing.TB) *SchemaProvider { 42 | mock := &SchemaProvider{} 43 | mock.Mock.Test(t) 44 | 45 | t.Cleanup(func() { mock.AssertExpectations(t) }) 46 | 47 | return mock 48 | } 49 | -------------------------------------------------------------------------------- /core/schema/mocks/namespace_service.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v2.12.2. DO NOT EDIT. 2 | 3 | package mocks 4 | 5 | import ( 6 | context "context" 7 | 8 | namespace "github.com/raystack/stencil/core/namespace" 9 | mock "github.com/stretchr/testify/mock" 10 | 11 | testing "testing" 12 | ) 13 | 14 | // NamespaceService is an autogenerated mock type for the NamespaceService type 15 | type NamespaceService struct { 16 | mock.Mock 17 | } 18 | 19 | // Get provides a mock function with given fields: ctx, name 20 | func (_m *NamespaceService) Get(ctx context.Context, name string) (namespace.Namespace, error) { 21 | ret := _m.Called(ctx, name) 22 | 23 | var r0 namespace.Namespace 24 | if rf, ok := ret.Get(0).(func(context.Context, string) namespace.Namespace); ok { 25 | r0 = rf(ctx, name) 26 | } else { 27 | r0 = ret.Get(0).(namespace.Namespace) 28 | } 29 | 30 | var r1 error 31 | if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { 32 | r1 = rf(ctx, name) 33 | } else { 34 | r1 = ret.Error(1) 35 | } 36 | 37 | return r0, r1 38 | } 39 | 40 | // NewNamespaceService creates a new instance of NamespaceService. It also registers the testing.TB interface on the mock and a cleanup function to assert the mocks expectations. 41 | func NewNamespaceService(t testing.TB) *NamespaceService { 42 | mock := &NamespaceService{} 43 | mock.Mock.Test(t) 44 | 45 | t.Cleanup(func() { mock.AssertExpectations(t) }) 46 | 47 | return mock 48 | } 49 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | NAME="github.com/raystack/stencil" 2 | VERSION=$(shell git describe --always --tags 2>/dev/null) 3 | PROTON_COMMIT := "a6c7056fa80128145d00d5ee72f216c28578ec43" 4 | 5 | .PHONY: all build test clean dist vet proto install ui 6 | 7 | all: build 8 | 9 | build: ui ## Build the stencil binary 10 | go build -ldflags "-X config.Version=${VERSION}" ${NAME} 11 | 12 | test: ui ## Run the tests 13 | go test ./... -coverprofile=coverage.out 14 | 15 | coverage: ui ## Print code coverage 16 | go test -race -coverprofile coverage.txt -covermode=atomic ./... & go tool cover -html=coverage.out 17 | 18 | vet: ## Run the go vet tool 19 | go vet ./... 20 | 21 | lint: ## Run golang-ci lint 22 | golangci-lint run 23 | 24 | proto: ## Generate the protobuf files 25 | @echo " > generating protobuf from raystack/proton" 26 | @echo " > [info] make sure correct version of dependencies are installed using 'make install'" 27 | @buf generate https://github.com/raystack/proton/archive/${PROTON_COMMIT}.zip#strip_components=1 --template buf.gen.yaml --path raystack/stencil 28 | @echo " > protobuf compilation finished" 29 | 30 | clean: ## Clean the build artifacts 31 | rm -rf stencil dist/ ui/build/ 32 | 33 | ui: 34 | @echo " > generating ui build" 35 | @cd ui && $(MAKE) dep && $(MAKE) dist 36 | 37 | help: ## Display this help message 38 | @cat $(MAKEFILE_LIST) | grep -e "^[a-zA-Z_\-]*: *.*## *" | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' -------------------------------------------------------------------------------- /pkg/graph/graph.go: -------------------------------------------------------------------------------- 1 | package graph 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/emicklei/dot" 7 | "google.golang.org/protobuf/reflect/protodesc" 8 | "google.golang.org/protobuf/reflect/protoreflect" 9 | "google.golang.org/protobuf/types/descriptorpb" 10 | ) 11 | 12 | const ( 13 | NodeShape = "note" 14 | NodeStyle = "filled" 15 | NodeColor = "cornsilk" 16 | ) 17 | 18 | func GetProtoFileDependencyGraph(file *descriptorpb.FileDescriptorSet) (*dot.Graph, error) { 19 | files, err := protodesc.NewFiles(file) 20 | if err != nil { 21 | return nil, err 22 | } 23 | 24 | di := dot.NewGraph(dot.Directed) 25 | di.Attr("rankdir", "LR") 26 | 27 | files.RangeFiles(func(file protoreflect.FileDescriptor) bool { 28 | sourceNode := di.Node(fmt.Sprintf("%s\n%s", string(file.Package()), file.Path())) 29 | setDefaultAttributes(sourceNode) 30 | 31 | buildGraph(di, sourceNode, file) 32 | return true 33 | }) 34 | 35 | return di, nil 36 | } 37 | 38 | func buildGraph(di *dot.Graph, sourceNode dot.Node, file protoreflect.FileDescriptor) { 39 | for i := 0; i < file.Imports().Len(); i++ { 40 | imp := file.Imports().Get(i) 41 | destNode := di.Node(fmt.Sprintf("%s\n%s", string(imp.Package()), imp.Path())) 42 | setDefaultAttributes(destNode) 43 | 44 | di.Edge(sourceNode, destNode, "") 45 | } 46 | } 47 | 48 | func setDefaultAttributes(n dot.Node) { 49 | n.Attr("shape", NodeShape) 50 | n.Attr("style", NodeStyle) 51 | n.Attr("fillcolor", NodeColor) 52 | } 53 | -------------------------------------------------------------------------------- /internal/api/search.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/raystack/stencil/core/search" 8 | stencilv1beta1 "github.com/raystack/stencil/proto/raystack/stencil/v1beta1" 9 | ) 10 | 11 | func (a *API) Search(ctx context.Context, in *stencilv1beta1.SearchRequest) (*stencilv1beta1.SearchResponse, error) { 12 | searchReq := &search.SearchRequest{ 13 | NamespaceID: in.GetNamespaceId(), 14 | Query: in.GetQuery(), 15 | SchemaID: in.GetSchemaId(), 16 | } 17 | 18 | switch v := in.GetVersion().(type) { 19 | case *stencilv1beta1.SearchRequest_VersionId: 20 | searchReq.VersionID = v.VersionId 21 | case *stencilv1beta1.SearchRequest_History: 22 | searchReq.History = v.History 23 | } 24 | 25 | res, err := a.search.Search(ctx, searchReq) 26 | if err != nil { 27 | return nil, err 28 | } 29 | 30 | hits := make([]*stencilv1beta1.SearchHits, 0) 31 | for _, hit := range res.Hits { 32 | hits = append(hits, &stencilv1beta1.SearchHits{ 33 | SchemaId: hit.SchemaID, 34 | VersionId: hit.VersionID, 35 | Fields: hit.Fields, 36 | Types: hit.Types, 37 | NamespaceId: hit.NamespaceID, 38 | Path: fmt.Sprintf("/v1beta1/namespaces/%s/schemas/%s/versions/%d", hit.NamespaceID, hit.SchemaID, hit.VersionID), 39 | }) 40 | } 41 | return &stencilv1beta1.SearchResponse{ 42 | Hits: hits, 43 | Meta: &stencilv1beta1.SearchMeta{ 44 | Total: uint32(len(hits)), 45 | }, 46 | }, nil 47 | } 48 | -------------------------------------------------------------------------------- /formats/protobuf/testdata/forward/previous/1.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package a; 4 | 5 | enum TestEnum { 6 | UNSPECIFIED = 0; 7 | ONE = 1; 8 | } 9 | // Unchange message 10 | message One { 11 | string field_one = 1; 12 | } 13 | // This message will be deleted 14 | message WillBeDeleted { 15 | string one = 1; 16 | int64 two = 2; 17 | } 18 | // This enum will be deleted 19 | enum EnumWillBeDeleted { 20 | UNKNOWN = 0; 21 | TWO = 2; 22 | } 23 | // forward compatible message 24 | message Two { 25 | reserved 4,5; 26 | reserved "deleted_field"; 27 | string string_field = 1; 28 | int64 int_field = 2; 29 | TestEnum enum_field = 3; 30 | string going_to_delete = 6; 31 | enum CompatibleEnum { 32 | UNKNOWN = 0; 33 | ONE = 1; 34 | TWO = 2; 35 | } 36 | } 37 | 38 | message BreakingMessage { 39 | string name_change = 1; 40 | string number_change = 2; 41 | int64 num_exchange_a = 3; 42 | string num_exchange_b = 4; 43 | string kind_change = 5; 44 | TestEnum type_name_change = 6; 45 | One type_message_change = 7; 46 | string delete_without_reseve = 8; 47 | string delete_without_reseve_num = 9; 48 | string delete_without_reseve_name = 10; 49 | enum BreackingEnum { 50 | UNKNOWN = 0; 51 | NUMBER_CHANGE = 1; 52 | NAME_CHANGE = 3; 53 | DELETE_ENUM_WITHOUT_RESERVE = 4; 54 | DELETE_ENUM_WITHOUT_RESERVE_NUM = 5; 55 | DELETE_ENUM_WITHOUT_RESERVE_NAME = 6; 56 | } 57 | } 58 | message ChangeReserve { 59 | reserved 1 to 5, 8, 11; 60 | reserved "a", "b", "c"; 61 | } 62 | 63 | 64 | -------------------------------------------------------------------------------- /cmd/config.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/MakeNowJust/heredoc" 7 | "github.com/spf13/cobra" 8 | ) 9 | 10 | func configCmd(cdk *CDK) *cobra.Command { 11 | cmd := &cobra.Command{ 12 | Use: "config ", 13 | Short: "Manage stencil CLI configuration", 14 | } 15 | cmd.AddCommand(configInitCommand(cdk)) 16 | cmd.AddCommand(configListCommand(cdk)) 17 | return cmd 18 | } 19 | 20 | func configInitCommand(cdk *CDK) *cobra.Command { 21 | return &cobra.Command{ 22 | Use: "init", 23 | Short: "Initialize CLI configuration", 24 | Example: heredoc.Doc(` 25 | $ stencil config init 26 | `), 27 | Annotations: map[string]string{ 28 | "group": "core", 29 | }, 30 | RunE: func(cmd *cobra.Command, args []string) error { 31 | if err := cdk.Config.Init(&ClientConfig{}); err != nil { 32 | return err 33 | } 34 | 35 | fmt.Printf("Config created\n") 36 | return nil 37 | }, 38 | } 39 | } 40 | 41 | func configListCommand(cdk *CDK) *cobra.Command { 42 | var cmd = &cobra.Command{ 43 | Use: "list", 44 | Short: "List client configuration settings", 45 | Example: heredoc.Doc(` 46 | $ stencil config list 47 | `), 48 | Annotations: map[string]string{ 49 | "group": "core", 50 | }, 51 | RunE: func(cmd *cobra.Command, args []string) error { 52 | data, err := cdk.Config.View() 53 | if err != nil { 54 | return ErrClientConfigNotFound 55 | } 56 | 57 | fmt.Println(data) 58 | return nil 59 | }, 60 | } 61 | return cmd 62 | } 63 | -------------------------------------------------------------------------------- /internal/api/mocks/search_service.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v2.12.2. DO NOT EDIT. 2 | 3 | package mocks 4 | 5 | import ( 6 | context "context" 7 | 8 | search "github.com/raystack/stencil/core/search" 9 | mock "github.com/stretchr/testify/mock" 10 | 11 | testing "testing" 12 | ) 13 | 14 | // SearchService is an autogenerated mock type for the SearchService type 15 | type SearchService struct { 16 | mock.Mock 17 | } 18 | 19 | // Search provides a mock function with given fields: ctx, req 20 | func (_m *SearchService) Search(ctx context.Context, req *search.SearchRequest) (*search.SearchResponse, error) { 21 | ret := _m.Called(ctx, req) 22 | 23 | var r0 *search.SearchResponse 24 | if rf, ok := ret.Get(0).(func(context.Context, *search.SearchRequest) *search.SearchResponse); ok { 25 | r0 = rf(ctx, req) 26 | } else { 27 | if ret.Get(0) != nil { 28 | r0 = ret.Get(0).(*search.SearchResponse) 29 | } 30 | } 31 | 32 | var r1 error 33 | if rf, ok := ret.Get(1).(func(context.Context, *search.SearchRequest) error); ok { 34 | r1 = rf(ctx, req) 35 | } else { 36 | r1 = ret.Error(1) 37 | } 38 | 39 | return r0, r1 40 | } 41 | 42 | // NewSearchService creates a new instance of SearchService. It also registers the testing.TB interface on the mock and a cleanup function to assert the mocks expectations. 43 | func NewSearchService(t testing.TB) *SearchService { 44 | mock := &SearchService{} 45 | mock.Mock.Test(t) 46 | 47 | t.Cleanup(func() { mock.AssertExpectations(t) }) 48 | 49 | return mock 50 | } 51 | -------------------------------------------------------------------------------- /clients/java/src/main/java/io/odpf/stencil/client/ClassLoadStencilClient.java: -------------------------------------------------------------------------------- 1 | package org.raystack.stencil.client; 2 | 3 | import com.google.protobuf.Descriptors; 4 | 5 | import java.io.Serializable; 6 | import java.util.HashMap; 7 | import java.util.Map; 8 | 9 | /** 10 | * {@link StencilClient} implementation that can fetch descriptor from Protobuf Descritor classes in classpath 11 | */ 12 | public class ClassLoadStencilClient implements Serializable, StencilClient { 13 | 14 | transient private Map descriptorMap; 15 | 16 | public ClassLoadStencilClient() { 17 | } 18 | 19 | @Override 20 | public Descriptors.Descriptor get(String className) { 21 | if (descriptorMap == null) { 22 | descriptorMap = new HashMap<>(); 23 | } 24 | if (!descriptorMap.containsKey(className)) { 25 | try { 26 | Class protoClass = Class.forName(className); 27 | descriptorMap.put(className, (Descriptors.Descriptor) protoClass.getMethod("getDescriptor").invoke(null)); 28 | } catch (ReflectiveOperationException ignored) { 29 | 30 | } 31 | } 32 | return descriptorMap.get(className); 33 | } 34 | 35 | @Override 36 | public Map getAll() { 37 | throw new UnsupportedOperationException(); 38 | } 39 | 40 | @Override 41 | public void close() { 42 | } 43 | 44 | @Override 45 | public void refresh() { 46 | throw new UnsupportedOperationException(); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /clients/java/src/test/java/io/odpf/stencil/client/ClassLoadStencilClientTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.stencil.client; 2 | 3 | import com.google.protobuf.Descriptors; 4 | import org.raystack.stencil.StencilClientFactory; 5 | import org.junit.Test; 6 | 7 | import java.io.ByteArrayOutputStream; 8 | import java.io.IOException; 9 | import java.io.ObjectOutputStream; 10 | 11 | import static org.junit.Assert.assertNotNull; 12 | import static org.junit.Assert.assertNull; 13 | 14 | public class ClassLoadStencilClientTest { 15 | 16 | private static final String LOOKUP_KEY = "org.raystack.stencil.TestMessage"; 17 | 18 | @Test 19 | public void getDescriptorFromClassPath() { 20 | StencilClient c = StencilClientFactory.getClient(); 21 | Descriptors.Descriptor desc = c.get(LOOKUP_KEY); 22 | assertNotNull(desc); 23 | } 24 | 25 | @Test 26 | public void ClassNotPresent() { 27 | StencilClient c = StencilClientFactory.getClient(); 28 | Descriptors.Descriptor dsc = c.get("non_existent_proto"); 29 | assertNull(dsc); 30 | } 31 | 32 | @Test 33 | public void shouldBeSerializable() throws IOException { 34 | serializeObject(StencilClientFactory.getClient()); 35 | } 36 | 37 | public static byte[] serializeObject(Object o) throws IOException { 38 | try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); 39 | ObjectOutputStream oos = new ObjectOutputStream(baos)) { 40 | oos.writeObject(o); 41 | oos.flush(); 42 | return baos.toByteArray(); 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /cmd/download.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/MakeNowJust/heredoc" 8 | "github.com/raystack/salt/cli/printer" 9 | "github.com/spf13/cobra" 10 | ) 11 | 12 | func downloadSchemaCmd(cdk *CDK) *cobra.Command { 13 | var output, namespaceID string 14 | var version int32 15 | var data []byte 16 | 17 | cmd := &cobra.Command{ 18 | Use: "download ", 19 | Short: "Download a schema", 20 | Args: cobra.ExactArgs(1), 21 | Example: heredoc.Doc(` 22 | $ stencil schema download customer -n=raystack --version 1 23 | `), 24 | RunE: func(cmd *cobra.Command, args []string) error { 25 | spinner := printer.Spin("") 26 | defer spinner.Stop() 27 | client, cancel, err := createClient(cmd, cdk) 28 | if err != nil { 29 | return err 30 | } 31 | defer cancel() 32 | 33 | data, _, err = fetchSchemaAndMeta(client, version, namespaceID, args[0]) 34 | if err != nil { 35 | return err 36 | } 37 | spinner.Stop() 38 | 39 | err = os.WriteFile(output, data, 0666) 40 | if err != nil { 41 | return err 42 | } 43 | 44 | fmt.Printf("%s Schema successfully written to %s\n", printer.Green(printer.Icon("success")), output) 45 | return nil 46 | }, 47 | } 48 | 49 | cmd.Flags().StringVarP(&namespaceID, "namespace", "n", "", "Parent namespace ID") 50 | cmd.MarkFlagRequired("namespace") 51 | 52 | cmd.Flags().Int32VarP(&version, "version", "v", 0, "Version of the schema") 53 | 54 | cmd.Flags().StringVarP(&output, "output", "o", "", "Path to the output file") 55 | cmd.MarkFlagRequired("output") 56 | 57 | return cmd 58 | } 59 | -------------------------------------------------------------------------------- /clients/java/src/test/java/io/odpf/stencil/URLStencilClientTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.stencil; 2 | 3 | import com.github.tomakehurst.wiremock.WireMockServer; 4 | import com.github.tomakehurst.wiremock.core.WireMockConfiguration; 5 | import com.google.protobuf.Descriptors; 6 | import org.raystack.stencil.client.StencilClient; 7 | import org.raystack.stencil.config.StencilConfig; 8 | import org.junit.After; 9 | import org.junit.Before; 10 | import org.junit.Test; 11 | 12 | import java.io.IOException; 13 | import java.util.Map; 14 | 15 | import static org.junit.Assert.assertNotNull; 16 | 17 | public class URLStencilClientTest { 18 | private WireMockServer wireMockServer; 19 | @Before 20 | public void setup() { 21 | WireMockConfiguration config = new WireMockConfiguration(); 22 | config = config.withRootDirectory("src/test/resources/").port(8082); 23 | wireMockServer = new WireMockServer(config); 24 | wireMockServer.start(); 25 | } 26 | 27 | @After 28 | public void tearDown() { 29 | wireMockServer.stop(); 30 | } 31 | 32 | @Test 33 | public void downloadFile() throws IOException { 34 | String url = "http://localhost:8082/descriptors.bin"; 35 | StencilClient c = StencilClientFactory.getClient(url, StencilConfig.builder().build()); 36 | Map descMap = c.getAll(); 37 | assertNotNull(descMap); 38 | Descriptors.Descriptor desc = c.get("org.raystack.stencil.TestMessage"); 39 | assertNotNull(desc); 40 | c.refresh(); 41 | c.close(); 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /clients/java/src/test/java/io/odpf/stencil/MultiURLStencilClientTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.stencil; 2 | 3 | import com.github.tomakehurst.wiremock.WireMockServer; 4 | import com.github.tomakehurst.wiremock.core.WireMockConfiguration; 5 | import com.google.protobuf.Descriptors; 6 | import org.raystack.stencil.client.StencilClient; 7 | import org.raystack.stencil.config.StencilConfig; 8 | import org.junit.After; 9 | import org.junit.Before; 10 | import org.junit.Test; 11 | 12 | import java.util.ArrayList; 13 | import java.util.Map; 14 | 15 | import static org.junit.Assert.assertNotNull; 16 | 17 | public class MultiURLStencilClientTest { 18 | private WireMockServer wireMockServer; 19 | 20 | @Before 21 | public void setup() { 22 | WireMockConfiguration config = new WireMockConfiguration(); 23 | config = config.withRootDirectory("src/test/resources/").port(8082); 24 | wireMockServer = new WireMockServer(config); 25 | wireMockServer.start(); 26 | } 27 | 28 | @After 29 | public void tearDown() { 30 | wireMockServer.stop(); 31 | } 32 | 33 | @Test 34 | public void shouldReturnDescriptor() { 35 | ArrayList urls = new ArrayList(); 36 | urls.add("http://localhost:8082/descriptors.bin"); 37 | StencilClient c = StencilClientFactory.getClient(urls, StencilConfig.builder().build()); 38 | Map descMap = c.getAll(); 39 | assertNotNull(descMap); 40 | Descriptors.Descriptor desc = c.get("org.raystack.stencil.TestMessage"); 41 | assertNotNull(desc); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /clients/js/lib/multi_url_stencil.js: -------------------------------------------------------------------------------- 1 | const Stencil = require('./stencil'); 2 | 3 | /** 4 | * @typedef {import("./stencil").Options} Options 5 | */ 6 | 7 | /** 8 | * MultiURLStencil Client class 9 | */ 10 | class MultiURLStencil { 11 | /** 12 | * 13 | * @param {string[]} urls 14 | * @param {Options} options - Options for stencil client 15 | */ 16 | constructor(urls, options) { 17 | this.urls = urls; 18 | this.options = options; 19 | this.clients = []; 20 | } 21 | 22 | async init() { 23 | this.clients = await Promise.all( 24 | this.urls.map((url) => Stencil.getInstance(url, this.options)) 25 | ); 26 | } 27 | 28 | /** 29 | * Clears any active timers if present 30 | */ 31 | close() { 32 | this.clients.forEach((client) => client.close()); 33 | } 34 | 35 | /** 36 | * @param {string} protoName 37 | * @returns {protobuf.Type} 38 | */ 39 | getType(protoName) { 40 | let proto; 41 | for (let i = 0; i < this.clients.length; i += 1) { 42 | const client = this.clients[i]; 43 | try { 44 | proto = client.getType(protoName); 45 | return proto; 46 | } catch (e) { 47 | // do nothing 48 | } 49 | } 50 | throw new Error(`no such type: ${protoName}`); 51 | } 52 | 53 | /** 54 | * 55 | * @param {string[]} urls 56 | * @param {Options} options - Options for stencil client 57 | */ 58 | static async getInstance(urls, options) { 59 | const stencil = new MultiURLStencil(urls, options); 60 | await stencil.init(); 61 | return stencil; 62 | } 63 | } 64 | 65 | module.exports = MultiURLStencil; 66 | -------------------------------------------------------------------------------- /clients/java/src/main/java/io/odpf/stencil/http/RemoteFileImpl.java: -------------------------------------------------------------------------------- 1 | package org.raystack.stencil.http; 2 | 3 | import org.apache.http.HttpEntity; 4 | import org.apache.http.HttpResponse; 5 | import org.apache.http.client.ClientProtocolException; 6 | import org.apache.http.client.ResponseHandler; 7 | import org.apache.http.client.methods.HttpGet; 8 | import org.apache.http.impl.client.CloseableHttpClient; 9 | import org.apache.http.util.EntityUtils; 10 | 11 | import java.io.IOException; 12 | 13 | public class RemoteFileImpl implements RemoteFile, ResponseHandler { 14 | private CloseableHttpClient closeableHttpClient; 15 | 16 | public RemoteFileImpl(CloseableHttpClient httpClient) { 17 | this.closeableHttpClient = httpClient; 18 | } 19 | 20 | public byte[] fetch(String url) throws IOException { 21 | HttpGet httpget = new HttpGet(url); 22 | byte[] responseBody; 23 | responseBody = closeableHttpClient.execute(httpget, this); 24 | return responseBody; 25 | } 26 | 27 | @Override 28 | public void close() throws IOException { 29 | closeableHttpClient.close(); 30 | } 31 | 32 | @Override 33 | public byte[] handleResponse(HttpResponse response) throws IOException { 34 | int status = response.getStatusLine().getStatusCode(); 35 | if (status >= 200 && status < 300) { 36 | HttpEntity entity = response.getEntity(); 37 | return entity != null ? EntityUtils.toByteArray(entity) : null; 38 | } else { 39 | throw new ClientProtocolException("Unexpected response status: " + status); 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /cmd/edit.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/MakeNowJust/heredoc" 8 | "github.com/raystack/salt/cli/printer" 9 | stencilv1beta1 "github.com/raystack/stencil/proto/raystack/stencil/v1beta1" 10 | "github.com/spf13/cobra" 11 | ) 12 | 13 | func editSchemaCmd(cdk *CDK) *cobra.Command { 14 | var comp, namespaceID string 15 | var req stencilv1beta1.UpdateSchemaMetadataRequest 16 | 17 | cmd := &cobra.Command{ 18 | Use: "edit", 19 | Short: "Edit a schema", 20 | Args: cobra.ExactArgs(1), 21 | Example: heredoc.Doc(` 22 | $ stencil schema edit booking -n raystack -c COMPATIBILITY_BACKWARD 23 | `), 24 | RunE: func(cmd *cobra.Command, args []string) error { 25 | spinner := printer.Spin("") 26 | defer spinner.Stop() 27 | 28 | client, cancel, err := createClient(cmd, cdk) 29 | if err != nil { 30 | return err 31 | } 32 | defer cancel() 33 | 34 | schemaID := args[0] 35 | 36 | req.NamespaceId = namespaceID 37 | req.SchemaId = schemaID 38 | req.Compatibility = stencilv1beta1.Schema_Compatibility(stencilv1beta1.Schema_Compatibility_value[comp]) 39 | 40 | _, err = client.UpdateSchemaMetadata(context.Background(), &req) 41 | if err != nil { 42 | return err 43 | } 44 | 45 | spinner.Stop() 46 | fmt.Printf("Schema successfully updated") 47 | return nil 48 | }, 49 | } 50 | 51 | cmd.Flags().StringVarP(&namespaceID, "namespace", "n", "", "Parent namespace ID") 52 | cmd.MarkFlagRequired("namespace") 53 | 54 | cmd.Flags().StringVarP(&comp, "comp", "c", "", "Schema compatibility") 55 | cmd.MarkFlagRequired("comp") 56 | 57 | return cmd 58 | } 59 | -------------------------------------------------------------------------------- /docs/src/css/theme.css: -------------------------------------------------------------------------------- 1 | /** 2 | * Any CSS included here will be global. The classic template 3 | * bundles Infima by default. Infima is a CSS framework designed to 4 | * work well for content-centric websites. 5 | */ 6 | 7 | @import url("https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700&display=swap"); 8 | 9 | /* You can override the default Infima variables here. */ 10 | :root { 11 | --ifm-color-primary: #4d4dcb; 12 | --ifm-color-primary-dark: #3939c3; 13 | --ifm-color-primary-darker: #3636b8; 14 | --ifm-color-primary-darkest: #2c2c98; 15 | --ifm-color-primary-light: #6363d1; 16 | --ifm-color-primary-lighter: #6e6ed4; 17 | --ifm-color-primary-lightest: #8e8ede; 18 | --ifm-code-font-size: 90%; 19 | --ifm-font-family-base: "Inter", sans-serif; 20 | } 21 | 22 | .docusaurus-highlight-code-line { 23 | background-color: rgba(0, 0, 0, 0.1); 24 | display: block; 25 | margin: 0 calc(-1 * var(--ifm-pre-padding)); 26 | padding: 0 var(--ifm-pre-padding); 27 | } 28 | 29 | html[data-theme="dark"] .docusaurus-highlight-code-line { 30 | background-color: rgba(0, 0, 0, 0.3); 31 | } 32 | 33 | html[data-theme="light"] { 34 | --main-bg-color: var(--ifm-color-primary); 35 | --light-bg-color: transparent; 36 | --dark-bg-color: #f8f6f0; 37 | } 38 | 39 | html[data-theme="dark"] { 40 | --main-bg-color: var(--ifm-color-primary); 41 | --light-bg-color: transparent; 42 | --dark-bg-color: #131313; 43 | } 44 | 45 | .menu { 46 | font-size: 90%; 47 | } 48 | 49 | .markdown h1 { 50 | font-size: 2.5rem; 51 | } 52 | .markdown h2 { 53 | font-size: 1.7rem; 54 | } 55 | 56 | .markdown h3 { 57 | font-size: 1.4rem; 58 | } 59 | -------------------------------------------------------------------------------- /clients/java/src/test/java/io/odpf/stencil/http/RetryHttpClientTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.stencil.http; 2 | 3 | import com.github.tomakehurst.wiremock.junit.WireMockRule; 4 | import org.raystack.stencil.config.StencilConfig; 5 | 6 | import org.apache.http.Header; 7 | import org.apache.http.HttpHeaders; 8 | import org.apache.http.client.methods.HttpGet; 9 | import org.apache.http.impl.client.CloseableHttpClient; 10 | import org.apache.http.message.BasicHeader; 11 | import org.junit.Rule; 12 | import org.junit.Test; 13 | 14 | import java.io.IOException; 15 | import java.util.ArrayList; 16 | import java.util.List; 17 | 18 | import static com.github.tomakehurst.wiremock.client.WireMock.*; 19 | 20 | 21 | public class RetryHttpClientTest { 22 | 23 | @Rule 24 | public WireMockRule service = new WireMockRule(8081); 25 | 26 | @Test 27 | public void shouldUseAuthenticationBearerTokenFromStencilConfig() throws IOException { 28 | String token = "test-token"; 29 | 30 | service.stubFor(any(anyUrl()) 31 | .willReturn(aResponse() 32 | .withStatus(200)) 33 | ); 34 | Header authHeader = new BasicHeader(HttpHeaders.AUTHORIZATION, "Bearer " + token); 35 | List
headers = new ArrayList
(); 36 | headers.add(authHeader); 37 | 38 | CloseableHttpClient httpClient = new RetryHttpClient().create(StencilConfig.builder().fetchHeaders(headers).build()); 39 | httpClient.execute(new HttpGet(service.url("/test/stencil/auth/header"))); 40 | 41 | verify(getRequestedFor(anyUrl()).withHeader(HttpHeaders.AUTHORIZATION, equalTo("Bearer " + token))); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /core/schema/mocks/schema_cache.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v2.12.2. DO NOT EDIT. 2 | 3 | package mocks 4 | 5 | import ( 6 | mock "github.com/stretchr/testify/mock" 7 | 8 | testing "testing" 9 | ) 10 | 11 | // SchemaCache is an autogenerated mock type for the Cache type 12 | type SchemaCache struct { 13 | mock.Mock 14 | } 15 | 16 | // Get provides a mock function with given fields: _a0 17 | func (_m *SchemaCache) Get(_a0 interface{}) (interface{}, bool) { 18 | ret := _m.Called(_a0) 19 | 20 | var r0 interface{} 21 | if rf, ok := ret.Get(0).(func(interface{}) interface{}); ok { 22 | r0 = rf(_a0) 23 | } else { 24 | if ret.Get(0) != nil { 25 | r0 = ret.Get(0).(interface{}) 26 | } 27 | } 28 | 29 | var r1 bool 30 | if rf, ok := ret.Get(1).(func(interface{}) bool); ok { 31 | r1 = rf(_a0) 32 | } else { 33 | r1 = ret.Get(1).(bool) 34 | } 35 | 36 | return r0, r1 37 | } 38 | 39 | // Set provides a mock function with given fields: _a0, _a1, _a2 40 | func (_m *SchemaCache) Set(_a0 interface{}, _a1 interface{}, _a2 int64) bool { 41 | ret := _m.Called(_a0, _a1, _a2) 42 | 43 | var r0 bool 44 | if rf, ok := ret.Get(0).(func(interface{}, interface{}, int64) bool); ok { 45 | r0 = rf(_a0, _a1, _a2) 46 | } else { 47 | r0 = ret.Get(0).(bool) 48 | } 49 | 50 | return r0 51 | } 52 | 53 | // NewSchemaCache creates a new instance of SchemaCache. It also registers the testing.TB interface on the mock and a cleanup function to assert the mocks expectations. 54 | func NewSchemaCache(t testing.TB) *SchemaCache { 55 | mock := &SchemaCache{} 56 | mock.Mock.Test(t) 57 | 58 | t.Cleanup(func() { mock.AssertExpectations(t) }) 59 | 60 | return mock 61 | } 62 | -------------------------------------------------------------------------------- /docs/src/core/GridBlock.js: -------------------------------------------------------------------------------- 1 | import classNames from 'classnames'; 2 | import * as React from 'react'; 3 | 4 | class GridBlock extends React.Component { 5 | renderBlock(origBlock) { 6 | const blockDefaults = { 7 | imageAlign: 'left', 8 | }; 9 | 10 | const block = { 11 | ...blockDefaults, 12 | ...origBlock, 13 | }; 14 | 15 | const blockClasses = classNames('blockElement', this.props.className, { 16 | alignCenter: this.props.align === 'center', 17 | alignRight: this.props.align === 'right', 18 | fourByGridBlock: this.props.layout === 'fourColumn', 19 | threeByGridBlock: this.props.layout === 'threeColumn', 20 | twoByGridBlock: this.props.layout === 'twoColumn', 21 | }); 22 | 23 | return ( 24 |
25 |
26 | {this.renderBlockTitle(block.title)} 27 | {block.content} 28 |
29 |
30 | ); 31 | } 32 | 33 | renderBlockTitle(title) { 34 | if (!title) { 35 | return null; 36 | } 37 | 38 | return

{title}

; 39 | } 40 | 41 | render() { 42 | return ( 43 |
44 | {this.props.contents.map(this.renderBlock, this)} 45 |
46 | ); 47 | } 48 | } 49 | 50 | GridBlock.defaultProps = { 51 | align: 'left', 52 | contents: [], 53 | layout: 'twoColumn', 54 | }; 55 | 56 | export default GridBlock; 57 | -------------------------------------------------------------------------------- /internal/store/postgres/migrations/000001_initialize_schema.up.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS namespaces( 2 | id VARCHAR PRIMARY KEY, 3 | format VARCHAR, 4 | compatibility VARCHAR, 5 | description VARCHAR, 6 | created_at TIMESTAMP, 7 | updated_at TIMESTAMP 8 | ); 9 | 10 | CREATE TABLE IF NOT EXISTS schemas( 11 | id BIGSERIAL PRIMARY KEY, 12 | name VARCHAR NOT NULL, 13 | authority VARCHAR, 14 | format VARCHAR, 15 | compatibility VARCHAR, 16 | description VARCHAR, 17 | namespace_id VARCHAR NOT NULL, 18 | created_at TIMESTAMP, 19 | updated_at TIMESTAMP, 20 | CONSTRAINT fk_schemas_namespace_id FOREIGN KEY(namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE, 21 | CONSTRAINT schema_name_namespace_unique_idx UNIQUE (name, namespace_id) 22 | ); 23 | 24 | CREATE TABLE IF NOT EXISTS versions( 25 | id VARCHAR, 26 | version BIGINT, 27 | schema_id BIGINT, 28 | created_at TIMESTAMP, 29 | CONSTRAINT fk_versions_schema_id FOREIGN KEY(schema_id) REFERENCES schemas(id) ON DELETE CASCADE, 30 | CONSTRAINT schema_version_unique_idx UNIQUE (version, schema_id), 31 | CONSTRAINT schema_id_unique UNIQUE (id) 32 | ); 33 | 34 | CREATE TABLE IF NOT EXISTS schema_files( 35 | id VARCHAR, 36 | search_data JSONB, 37 | data bytea, 38 | created_at TIMESTAMP, 39 | updated_at TIMESTAMP, 40 | CONSTRAINT schema_files_id_unique_idx UNIQUE (id) 41 | ); 42 | 43 | CREATE TABLE IF NOT EXISTS versions_schema_files( 44 | version_id VARCHAR, 45 | schema_file_id VARCHAR, 46 | CONSTRAINT fk_versions_schema_files_version_id FOREIGN KEY(version_id) REFERENCES versions(id) ON DELETE CASCADE, 47 | CONSTRAINT fk_versions_schema_files_schema_file_id FOREIGN KEY(schema_file_id) REFERENCES schema_files(id) 48 | ); 49 | -------------------------------------------------------------------------------- /cmd/client.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/raystack/salt/config" 8 | stencilv1beta1 "github.com/raystack/stencil/proto/raystack/stencil/v1beta1" 9 | "github.com/spf13/cobra" 10 | "google.golang.org/grpc" 11 | "google.golang.org/grpc/credentials/insecure" 12 | ) 13 | 14 | type ClientConfig struct { 15 | Host string `yaml:"host" cmdx:"host"` 16 | } 17 | 18 | func createConnection(ctx context.Context, host string) (*grpc.ClientConn, error) { 19 | opts := []grpc.DialOption{ 20 | grpc.WithTransportCredentials(insecure.NewCredentials()), 21 | grpc.WithBlock(), 22 | } 23 | 24 | return grpc.DialContext(ctx, host, opts...) 25 | } 26 | 27 | func createClient(cmd *cobra.Command, cdk *CDK) (stencilv1beta1.StencilServiceClient, func(), error) { 28 | c, err := loadClientConfig(cmd, cdk.Config) 29 | if err != nil { 30 | return nil, nil, err 31 | } 32 | 33 | host := c.Host 34 | 35 | if host == "" { 36 | return nil, nil, ErrClientConfigHostNotFound 37 | } 38 | 39 | dialTimeoutCtx, dialCancel := context.WithTimeout(cmd.Context(), time.Second*2) 40 | conn, err := createConnection(dialTimeoutCtx, host) 41 | if err != nil { 42 | dialCancel() 43 | return nil, nil, err 44 | } 45 | 46 | cancel := func() { 47 | dialCancel() 48 | conn.Close() 49 | } 50 | 51 | client := stencilv1beta1.NewStencilServiceClient(conn) 52 | return client, cancel, nil 53 | } 54 | 55 | func loadClientConfig(cmd *cobra.Command, cmdxConfig *config.Loader) (*ClientConfig, error) { 56 | var clientConfig ClientConfig 57 | 58 | if err := cmdxConfig.Load( 59 | &clientConfig, 60 | ); err != nil { 61 | return nil, err 62 | } 63 | 64 | return &clientConfig, nil 65 | } 66 | -------------------------------------------------------------------------------- /cmd/root.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/MakeNowJust/heredoc" 5 | "github.com/raystack/salt/cli/commander" 6 | "github.com/raystack/salt/config" 7 | "github.com/spf13/cobra" 8 | ) 9 | 10 | type CDK struct { 11 | Config *config.Loader 12 | } 13 | 14 | // New root command 15 | func New() *cobra.Command { 16 | var cmd = &cobra.Command{ 17 | Use: "stencil [flags]", 18 | Short: "Schema registry", 19 | Long: "Schema registry to manage schemas efficiently.", 20 | SilenceUsage: true, 21 | SilenceErrors: true, 22 | Annotations: map[string]string{ 23 | "group": "core", 24 | "help:learn": heredoc.Doc(` 25 | Use 'stencil --help' for info about a command. 26 | Read the manual at https://raystack.github.io/stencil/ 27 | `), 28 | "help:feedback": heredoc.Doc(` 29 | Open an issue here https://github.com/raystack/stencil/issues 30 | `), 31 | }, 32 | } 33 | 34 | cdk := &CDK{ 35 | Config: config.NewLoader( 36 | config.WithAppConfig("stencil"), 37 | config.WithEnvPrefix("STENCIL"), 38 | config.WithFlags(cmd.Flags()), 39 | ), 40 | } 41 | 42 | cmd.AddCommand(ServerCommand()) 43 | cmd.AddCommand(configCmd(cdk)) 44 | cmd.AddCommand(NamespaceCmd(cdk)) 45 | cmd.AddCommand(SchemaCmd(cdk)) 46 | cmd.AddCommand(SearchCmd(cdk)) 47 | 48 | hooks := []commander.HookBehavior{ 49 | { 50 | Name: "client", 51 | Behavior: func(cmd *cobra.Command) { 52 | cmd.PersistentFlags().String("host", "", "Server host address") 53 | }, 54 | }, 55 | } 56 | 57 | // Help topics 58 | cmdr := commander.New( 59 | cmd, 60 | commander.WithTopics(envHelpTopics), 61 | commander.WithHooks(hooks), 62 | ) 63 | cmdr.Init() 64 | 65 | return cmd 66 | } 67 | -------------------------------------------------------------------------------- /formats/json/testdata/requiredProperties/removed.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "STAC Collection", 3 | "description": "These are the fields specific to a STAC Collection. All other fields are inherited from STAC Catalog.", 4 | "type": "object", 5 | "required": [ 6 | "stac_version", 7 | "type", 8 | "id", 9 | "license" 10 | ], 11 | "properties": { 12 | "stac_version": { 13 | "title": "STAC version", 14 | "type": "string", 15 | "const": "1.0.0" 16 | }, 17 | "stac_extensions": { 18 | "title": "STAC extensions", 19 | "type": "array", 20 | "uniqueItems": true, 21 | "items": { 22 | "title": "Reference to a JSON Schema", 23 | "type": "string", 24 | "format": "iri" 25 | } 26 | }, 27 | "type": { 28 | "title": "Type of STAC entity", 29 | "const": "Collection" 30 | }, 31 | "id": { 32 | "title": "Identifier", 33 | "type": "string", 34 | "minLength": 1 35 | }, 36 | "title": { 37 | "title": "Title", 38 | "type": "string" 39 | }, 40 | "description": { 41 | "title": "Description", 42 | "type": "string", 43 | "minLength": 1 44 | }, 45 | "keywords": { 46 | "title": "Keywords", 47 | "type": "array", 48 | "items": { 49 | "type": "string" 50 | } 51 | }, 52 | "license": { 53 | "title": "Collection License Name", 54 | "type": "string", 55 | "pattern": "^[\\w\\-\\.\\+]+$" 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /core/schema/schema.go: -------------------------------------------------------------------------------- 1 | package schema 2 | 3 | import "context" 4 | 5 | type Metadata struct { 6 | Authority string 7 | Format string 8 | Compatibility string 9 | } 10 | 11 | type SchemaInfo struct { 12 | ID string `json:"id"` 13 | Version int32 `json:"version"` 14 | Location string `json:"location"` 15 | } 16 | 17 | type SchemaFile struct { 18 | ID string 19 | Types []string 20 | Fields []string 21 | Data []byte 22 | } 23 | 24 | type Repository interface { 25 | Create(ctx context.Context, namespace string, schema string, metadata *Metadata, versionID string, schemaFile *SchemaFile) (version int32, err error) 26 | List(context.Context, string) ([]Schema, error) 27 | ListVersions(context.Context, string, string) ([]int32, error) 28 | Get(context.Context, string, string, int32) ([]byte, error) 29 | GetLatestVersion(context.Context, string, string) (int32, error) 30 | GetMetadata(context.Context, string, string) (*Metadata, error) 31 | UpdateMetadata(context.Context, string, string, *Metadata) (*Metadata, error) 32 | Delete(context.Context, string, string) error 33 | DeleteVersion(context.Context, string, string, int32) error 34 | } 35 | 36 | type ParsedSchema interface { 37 | IsBackwardCompatible(ParsedSchema) error 38 | IsForwardCompatible(ParsedSchema) error 39 | IsFullCompatible(ParsedSchema) error 40 | Format() string 41 | GetCanonicalValue() *SchemaFile 42 | } 43 | 44 | type Provider interface { 45 | ParseSchema(format string, data []byte) (ParsedSchema, error) 46 | } 47 | 48 | type Cache interface { 49 | Get(interface{}) (interface{}, bool) 50 | Set(interface{}, interface{}, int64) bool 51 | } 52 | 53 | type Schema struct { 54 | Name string 55 | Format string 56 | Compatibility string 57 | Authority string 58 | } 59 | -------------------------------------------------------------------------------- /formats/json/testdata/requiredProperties/prev.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "STAC Collection", 3 | "description": "These are the fields specific to a STAC Collection. All other fields are inherited from STAC Catalog.", 4 | "type": "object", 5 | "required": [ 6 | "stac_version", 7 | "type", 8 | "id", 9 | "description", 10 | "license" 11 | ], 12 | "properties": { 13 | "stac_version": { 14 | "title": "STAC version", 15 | "type": "string", 16 | "const": "1.0.0" 17 | }, 18 | "stac_extensions": { 19 | "title": "STAC extensions", 20 | "type": "array", 21 | "uniqueItems": true, 22 | "items": { 23 | "title": "Reference to a JSON Schema", 24 | "type": "string", 25 | "format": "iri" 26 | } 27 | }, 28 | "type": { 29 | "title": "Type of STAC entity", 30 | "const": "Collection" 31 | }, 32 | "id": { 33 | "title": "Identifier", 34 | "type": "string", 35 | "minLength": 1 36 | }, 37 | "title": { 38 | "title": "Title", 39 | "type": "string" 40 | }, 41 | "description": { 42 | "title": "Description", 43 | "type": "string", 44 | "minLength": 1 45 | }, 46 | "keywords": { 47 | "title": "Keywords", 48 | "type": "array", 49 | "items": { 50 | "type": "string" 51 | } 52 | }, 53 | "license": { 54 | "title": "Collection License Name", 55 | "type": "string", 56 | "pattern": "^[\\w\\-\\.\\+]+$" 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /formats/json/testdata/requiredProperties/added.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "STAC Collection", 3 | "description": "These are the fields specific to a STAC Collection. All other fields are inherited from STAC Catalog.", 4 | "type": "object", 5 | "required": [ 6 | "stac_version", 7 | "type", 8 | "id", 9 | "description", 10 | "license", 11 | "stac_extensions" 12 | ], 13 | "properties": { 14 | "stac_version": { 15 | "title": "STAC version", 16 | "type": "string", 17 | "const": "1.0.0" 18 | }, 19 | "stac_extensions": { 20 | "title": "STAC extensions", 21 | "type": "array", 22 | "uniqueItems": true, 23 | "items": { 24 | "title": "Reference to a JSON Schema", 25 | "type": "string", 26 | "format": "iri" 27 | } 28 | }, 29 | "type": { 30 | "title": "Type of STAC entity", 31 | "const": "Collection" 32 | }, 33 | "id": { 34 | "title": "Identifier", 35 | "type": "string", 36 | "minLength": 1 37 | }, 38 | "title": { 39 | "title": "Title", 40 | "type": "string" 41 | }, 42 | "description": { 43 | "title": "Description", 44 | "type": "string", 45 | "minLength": 1 46 | }, 47 | "keywords": { 48 | "title": "Keywords", 49 | "type": "array", 50 | "items": { 51 | "type": "string" 52 | } 53 | }, 54 | "license": { 55 | "title": "Collection License Name", 56 | "type": "string", 57 | "pattern": "^[\\w\\-\\.\\+]+$" 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /cmd/delete.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/MakeNowJust/heredoc" 8 | "github.com/raystack/salt/cli/printer" 9 | stencilv1beta1 "github.com/raystack/stencil/proto/raystack/stencil/v1beta1" 10 | "github.com/spf13/cobra" 11 | ) 12 | 13 | func deleteSchemaCmd(cdk *CDK) *cobra.Command { 14 | var namespaceID string 15 | var req stencilv1beta1.DeleteSchemaRequest 16 | var reqVer stencilv1beta1.DeleteVersionRequest 17 | var version int32 18 | 19 | cmd := &cobra.Command{ 20 | Use: "delete ", 21 | Short: "Delete a schema", 22 | Args: cobra.ExactArgs(1), 23 | Example: heredoc.Doc(` 24 | $ stencil schema delete booking -n raystack 25 | `), 26 | RunE: func(cmd *cobra.Command, args []string) error { 27 | spinner := printer.Spin("") 28 | defer spinner.Stop() 29 | 30 | client, cancel, err := createClient(cmd, cdk) 31 | if err != nil { 32 | return err 33 | } 34 | defer cancel() 35 | 36 | schemaID := args[0] 37 | 38 | if version == 0 { 39 | req.NamespaceId = namespaceID 40 | req.SchemaId = schemaID 41 | 42 | _, err = client.DeleteSchema(context.Background(), &req) 43 | if err != nil { 44 | return err 45 | } 46 | } else { 47 | reqVer.NamespaceId = namespaceID 48 | reqVer.SchemaId = schemaID 49 | reqVer.VersionId = version 50 | 51 | _, err = client.DeleteVersion(context.Background(), &reqVer) 52 | if err != nil { 53 | return err 54 | } 55 | } 56 | 57 | spinner.Stop() 58 | fmt.Printf("Schema successfully deleted") 59 | return nil 60 | }, 61 | } 62 | 63 | cmd.Flags().StringVarP(&namespaceID, "namespace", "n", "", "Parent namespace ID") 64 | cmd.MarkFlagRequired("namespace") 65 | 66 | cmd.Flags().Int32VarP(&version, "version", "v", 0, "Particular version to be deleted") 67 | 68 | return cmd 69 | } 70 | -------------------------------------------------------------------------------- /internal/store/errors.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "fmt" 5 | 6 | "google.golang.org/grpc/codes" 7 | "google.golang.org/grpc/status" 8 | ) 9 | 10 | type errKind int 11 | 12 | const ( 13 | _ errKind = iota 14 | unknown 15 | conflict 16 | noRows 17 | ) 18 | 19 | var ( 20 | //UnknownErr default sentinel error for storage layer 21 | UnknownErr = StorageErr{kind: unknown} 22 | //ConflictErr can used for contraint violations 23 | ConflictErr = StorageErr{kind: conflict} 24 | //NoRowsErr can be used to represent not found/no result 25 | NoRowsErr = StorageErr{kind: noRows} 26 | ) 27 | 28 | // StorageErr implements error interface. Used for storage layer. Consumers can check for this error type to inspect storage errors. 29 | type StorageErr struct { 30 | name string 31 | kind errKind 32 | err error 33 | } 34 | 35 | func (e StorageErr) Error() string { 36 | if e.err != nil { 37 | return e.err.Error() 38 | } 39 | return e.name 40 | } 41 | 42 | // GRPCStatus this is used by gateway interceptor to return appropriate http status code and message 43 | func (e StorageErr) GRPCStatus() *status.Status { 44 | if e.kind == noRows { 45 | return status.New(codes.NotFound, fmt.Sprintf("%s %s", e.name, "not found")) 46 | } 47 | if e.kind == conflict { 48 | return status.New(codes.AlreadyExists, fmt.Sprintf("%s %s", e.name, "resource already exists")) 49 | } 50 | return status.New(codes.Unknown, e.Error()) 51 | } 52 | 53 | // WithErr convenience function to override sentinel errors 54 | func (e StorageErr) WithErr(err error, name string) StorageErr { 55 | e.err = err 56 | e.name = name 57 | return e 58 | } 59 | 60 | func (e StorageErr) Unwrap() error { 61 | return e.err 62 | } 63 | 64 | func (e StorageErr) Is(err error) bool { 65 | sErr, ok := err.(StorageErr) 66 | if !ok { 67 | return false 68 | } 69 | return e.kind == sErr.kind 70 | } 71 | -------------------------------------------------------------------------------- /core/schema/compatibility.go: -------------------------------------------------------------------------------- 1 | package schema 2 | 3 | import "go.uber.org/multierr" 4 | 5 | type ValidationStrategy func(ParsedSchema, ParsedSchema) error 6 | type CompatibilityFn func(ParsedSchema, []ParsedSchema) error 7 | 8 | func validateLatest(strategy ValidationStrategy) CompatibilityFn { 9 | return func(ps1 ParsedSchema, ps2 []ParsedSchema) error { 10 | for _, prev := range ps2 { 11 | return strategy(ps1, prev) 12 | } 13 | return nil 14 | } 15 | } 16 | 17 | func validateAll(strategy ValidationStrategy) CompatibilityFn { 18 | return func(ps1 ParsedSchema, ps2 []ParsedSchema) error { 19 | var err error 20 | for _, prev := range ps2 { 21 | e := strategy(ps1, prev) 22 | err = multierr.Combine(err, e) 23 | } 24 | return err 25 | } 26 | } 27 | 28 | func backwardStrategy(current, prev ParsedSchema) error { 29 | return current.IsBackwardCompatible(prev) 30 | } 31 | 32 | func forwardStrategy(current, prev ParsedSchema) error { 33 | return current.IsForwardCompatible(prev) 34 | } 35 | 36 | func fullStrategy(current, prev ParsedSchema) error { 37 | return current.IsFullCompatible(prev) 38 | } 39 | 40 | func defaultCompatibilityFn(current ParsedSchema, prevs []ParsedSchema) error { 41 | return nil 42 | } 43 | 44 | func getCompatibilityChecker(compatibility string) CompatibilityFn { 45 | switch compatibility { 46 | case "COMPATIBILITY_BACKWARD": 47 | return validateLatest(backwardStrategy) 48 | case "COMPATIBILITY_BACKWARD_TRANSITIVE": 49 | return validateAll(backwardStrategy) 50 | case "COMPATIBILITY_FORWARD": 51 | return validateLatest(forwardStrategy) 52 | case "COMPATIBILITY_FORWARD_TRANSITIVE": 53 | return validateAll(forwardStrategy) 54 | case "COMPATIBILITY_FULL": 55 | return validateLatest(fullStrategy) 56 | case "COMPATIBILITY_FULL_TRANSITIVE": 57 | return validateAll(fullStrategy) 58 | default: 59 | return defaultCompatibilityFn 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /formats/avro/schema.go: -------------------------------------------------------------------------------- 1 | package avro 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/google/uuid" 7 | "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" 8 | av "github.com/hamba/avro" 9 | "github.com/raystack/stencil/core/schema" 10 | "go.uber.org/multierr" 11 | ) 12 | 13 | const avroFormat = "FORMAT_AVRO" 14 | 15 | type Schema struct { 16 | data []byte 17 | sc av.Schema 18 | } 19 | 20 | func (s *Schema) Format() string { 21 | return avroFormat 22 | } 23 | 24 | func (s *Schema) GetCanonicalValue() *schema.SchemaFile { 25 | fingerprint := s.sc.Fingerprint() 26 | id := uuid.NewSHA1(uuid.NameSpaceOID, fingerprint[:]) 27 | return &schema.SchemaFile{ 28 | ID: id.String(), 29 | Data: s.data, 30 | } 31 | } 32 | 33 | func (s *Schema) verify(against schema.ParsedSchema) (*Schema, error) { 34 | prev, ok := against.(*Schema) 35 | if s.Format() == against.Format() && ok { 36 | return prev, nil 37 | } 38 | return nil, &runtime.HTTPStatusError{HTTPStatus: 400, Err: fmt.Errorf("current and prev schema formats(%s, %s) are different", s.Format(), against.Format())} 39 | } 40 | 41 | // IsBackwardCompatible checks backward compatibility against given schema 42 | func (s *Schema) IsBackwardCompatible(against schema.ParsedSchema) error { 43 | prev, err := s.verify(against) 44 | if err != nil { 45 | return err 46 | } 47 | c := av.NewSchemaCompatibility() 48 | return c.Compatible(s.sc, prev.sc) 49 | } 50 | 51 | // IsForwardCompatible checks backward compatibility against given schema 52 | func (s *Schema) IsForwardCompatible(against schema.ParsedSchema) error { 53 | return against.IsBackwardCompatible(s) 54 | } 55 | 56 | // IsFullCompatible checks for forward compatibility 57 | func (s *Schema) IsFullCompatible(against schema.ParsedSchema) error { 58 | forwardErr := s.IsForwardCompatible(against) 59 | backwardErr := s.IsBackwardCompatible(against) 60 | return multierr.Combine(forwardErr, backwardErr) 61 | } 62 | -------------------------------------------------------------------------------- /formats/json/testdata/requiredProperties/modified.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "STAC Collection", 3 | "description": "These are the fields specific to a STAC Collection. All other fields are inherited from STAC Catalog.", 4 | "type": "object", 5 | "required": [ 6 | "stac_version", 7 | "type", 8 | "id", 9 | "description", 10 | "license" 11 | ], 12 | "properties": { 13 | "stac_version": { 14 | "title": "STAC version", 15 | "type": "string", 16 | "const": "1.0.0" 17 | }, 18 | "stac_extensions": { 19 | "title": "STAC extensions", 20 | "type": "array", 21 | "uniqueItems": true, 22 | "items": { 23 | "title": "Reference to a JSON Schema", 24 | "type": "string", 25 | "format": "iri" 26 | } 27 | }, 28 | "type": { 29 | "title": "Type of STAC entity", 30 | "const": "Collection" 31 | }, 32 | "id": { 33 | "title": "Identifier", 34 | "type": "string", 35 | "minLength": 1 36 | }, 37 | "title": { 38 | "title": "Title", 39 | "type": "string" 40 | }, 41 | "description": { 42 | "title": "Description", 43 | "type": "string", 44 | "minLength": 1 45 | }, 46 | "keywords": { 47 | "title": "Keywords", 48 | "type": "array", 49 | "items": { 50 | "type": "string" 51 | } 52 | }, 53 | "license": { 54 | "title": "Collection License Name", 55 | "type": "string", 56 | "pattern": "^[\\w\\-\\.\\+]+$" 57 | }, 58 | "extras": { 59 | "title": "extra values", 60 | "type": "string" 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /clients/python/README.md: -------------------------------------------------------------------------------- 1 | # Stencil Python client 2 | 3 | [![PyPI version](https://badge.fury.io/py/stencil-python-client.svg)](https://pypi.org/project/stencil-python-client) 4 | 5 | Stencil Python client package provides a store to lookup protobuf descriptors and options to keep the protobuf descriptors upto date. 6 | 7 | It has following features 8 | - Deserialize protobuf messages directly by specifying protobuf message name 9 | - Ability to refresh protobuf descriptors in specified intervals 10 | - Support to download descriptors from multiple urls 11 | 12 | 13 | ## Requirements 14 | 15 | - Python 3.7+ 16 | 17 | ## Installation 18 | 19 | Install it via git reference 20 | ``` 21 | stencil-python-client = { git = "git+https://github.com/raystack/stencil.git", subdirectory = "clients/python"} 22 | ``` 23 | 24 | Then import the stencil package into your own code as mentioned below 25 | ```python 26 | from raystack import stencil 27 | ``` 28 | 29 | ## Usage 30 | 31 | ### Creating a client 32 | 33 | ```python 34 | from raystack import stencil 35 | 36 | url = "http://url/to/proto/descriptorset/file" 37 | client = stencil.Client(url) 38 | ``` 39 | 40 | ### Creating a multiURLClient 41 | 42 | ```python 43 | from raystack import stencil 44 | 45 | urls = ["http://urlA", "http://urlB"] 46 | client = stencil.MultiUrlClient(urls) 47 | ``` 48 | 49 | ### Get Descriptor 50 | ```python 51 | from raystack import stencil 52 | 53 | url = "http://url/to/proto/descriptorset/file" 54 | client = stencil.Client(url) 55 | client.get_descriptor("google.protobuf.DescriptorProto") 56 | ``` 57 | 58 | ### Parse protobuf message. 59 | ```python 60 | from raystack import stencil 61 | 62 | url = "http://url/to/proto/descriptorset/file" 63 | client = stencil.Client(url) 64 | 65 | data = "" 66 | desc = client.parse("google.protobuf.DescriptorProto", data) 67 | ``` 68 | 69 | Refer to [stencil documentation](https://raystack.gitbook.io/stencil/) for more information what you can do in stencil. 70 | -------------------------------------------------------------------------------- /clients/go/go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 2 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 4 | github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= 5 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 6 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 7 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 8 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 9 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 10 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 11 | github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= 12 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 13 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= 14 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 15 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 16 | google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= 17 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 18 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 19 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 20 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 21 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 22 | -------------------------------------------------------------------------------- /clients/go/store.go: -------------------------------------------------------------------------------- 1 | package stencil 2 | 3 | import ( 4 | "io" 5 | "sync" 6 | "time" 7 | ) 8 | 9 | type loaderFunc func(string) (*Resolver, error) 10 | type timer struct { 11 | ticker *time.Ticker 12 | done chan bool 13 | } 14 | 15 | func (t *timer) Close() error { 16 | t.ticker.Stop() 17 | t.done <- true 18 | return nil 19 | } 20 | 21 | func setInterval(d time.Duration, f func(), waitForReader <-chan bool) io.Closer { 22 | ticker := time.NewTicker(d) 23 | done := make(chan bool) 24 | go (func() { 25 | for { 26 | select { 27 | case <-done: 28 | return 29 | case <-ticker.C: 30 | // wait for access 31 | refresh := <-waitForReader 32 | if refresh { 33 | f() 34 | } 35 | } 36 | } 37 | })() 38 | return &timer{ticker: ticker, done: done} 39 | } 40 | 41 | type store struct { 42 | autoRefresh bool 43 | timer io.Closer 44 | access chan bool 45 | loader loaderFunc 46 | url string 47 | data *Resolver 48 | lock sync.RWMutex 49 | } 50 | 51 | func (s *store) refresh() { 52 | val, err := s.loader(s.url) 53 | if err == nil { 54 | s.lock.Lock() 55 | defer s.lock.Unlock() 56 | s.data = val 57 | } 58 | } 59 | 60 | func (s *store) notify() { 61 | select { 62 | case s.access <- true: 63 | default: 64 | } 65 | } 66 | 67 | func (s *store) getResolver() (*Resolver, bool) { 68 | s.notify() 69 | s.lock.RLock() 70 | defer s.lock.RUnlock() 71 | return s.data, s.data != nil 72 | } 73 | 74 | func (s *store) Close() { 75 | close(s.access) 76 | if s.timer != nil { 77 | s.timer.Close() 78 | } 79 | } 80 | 81 | func newStore(url string, options Options) (*store, error) { 82 | loader := options.RefreshStrategy.getLoader(options) 83 | s := &store{loader: loader, access: make(chan bool), url: url, autoRefresh: options.AutoRefresh} 84 | val, err := loader(url) 85 | if err != nil { 86 | return s, err 87 | } 88 | s.data = val 89 | if options.AutoRefresh { 90 | s.timer = setInterval(options.RefreshInterval, s.refresh, s.access) 91 | } 92 | return s, nil 93 | } 94 | -------------------------------------------------------------------------------- /cmd/server.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/MakeNowJust/heredoc" 5 | "github.com/raystack/stencil/config" 6 | "github.com/raystack/stencil/internal/server" 7 | "github.com/raystack/stencil/internal/store/postgres" 8 | "github.com/spf13/cobra" 9 | 10 | // Importing postgres driver 11 | _ "github.com/golang-migrate/migrate/v4/database/postgres" 12 | _ "github.com/golang-migrate/migrate/v4/source/file" 13 | ) 14 | 15 | func ServerCommand() *cobra.Command { 16 | cmd := &cobra.Command{ 17 | Use: "server ", 18 | Aliases: []string{"s"}, 19 | Short: "Server management", 20 | Long: "Server management commands.", 21 | Example: heredoc.Doc(` 22 | $ stencil server start 23 | $ stencil server start -c ./config.yaml 24 | $ stencil server migrate 25 | $ stencil server migrate -c ./config.yaml 26 | `), 27 | } 28 | 29 | cmd.AddCommand(startCommand()) 30 | cmd.AddCommand(migrateCommand()) 31 | 32 | return cmd 33 | } 34 | 35 | func startCommand() *cobra.Command { 36 | var configFile string 37 | 38 | cmd := &cobra.Command{ 39 | Use: "start", 40 | Aliases: []string{"s"}, 41 | Short: "Start the server", 42 | RunE: func(cmd *cobra.Command, args []string) error { 43 | cfg, err := config.Load(configFile) 44 | if err != nil { 45 | return err 46 | } 47 | server.Start(cfg) 48 | return nil 49 | }, 50 | } 51 | 52 | cmd.Flags().StringVarP(&configFile, "config", "c", "./config.yaml", "Config file path") 53 | return cmd 54 | } 55 | 56 | func migrateCommand() *cobra.Command { 57 | var configFile string 58 | 59 | cmd := &cobra.Command{ 60 | Use: "migrate", 61 | Short: "Run database migrations", 62 | RunE: func(cmd *cobra.Command, args []string) error { 63 | cfg, err := config.Load(configFile) 64 | if err != nil { 65 | return err 66 | } 67 | 68 | if err := postgres.Migrate(cfg.DB.ConnectionString); err != nil { 69 | return err 70 | } 71 | 72 | return nil 73 | }, 74 | } 75 | 76 | cmd.Flags().StringVarP(&configFile, "config", "c", "./config.yaml", "Config file path") 77 | return cmd 78 | } 79 | -------------------------------------------------------------------------------- /cmd/schema.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | 6 | stencilv1beta1 "github.com/raystack/stencil/proto/raystack/stencil/v1beta1" 7 | "github.com/spf13/cobra" 8 | ) 9 | 10 | func SchemaCmd(cdk *CDK) *cobra.Command { 11 | cmd := &cobra.Command{ 12 | Use: "schema", 13 | Aliases: []string{"schemas"}, 14 | Short: "Manage schemas", 15 | Long: "Work with schemas.", 16 | Annotations: map[string]string{ 17 | "group": "core", 18 | "client": "true", 19 | }, 20 | } 21 | 22 | cmd.AddCommand(createSchemaCmd(cdk)) 23 | cmd.AddCommand(listSchemaCmd(cdk)) 24 | cmd.AddCommand(infoSchemaCmd(cdk)) 25 | cmd.AddCommand(versionSchemaCmd(cdk)) 26 | cmd.AddCommand(printSchemaCmd(cdk)) 27 | cmd.AddCommand(downloadSchemaCmd(cdk)) 28 | cmd.AddCommand(checkSchemaCmd(cdk)) 29 | cmd.AddCommand(editSchemaCmd(cdk)) 30 | cmd.AddCommand(deleteSchemaCmd(cdk)) 31 | cmd.AddCommand(diffSchemaCmd(cdk)) 32 | cmd.AddCommand(graphSchemaCmd(cdk)) 33 | 34 | return cmd 35 | } 36 | 37 | func fetchSchemaAndMeta(client stencilv1beta1.StencilServiceClient, version int32, namespaceID, schemaID string) ([]byte, *stencilv1beta1.GetSchemaMetadataResponse, error) { 38 | var req stencilv1beta1.GetSchemaRequest 39 | var reqLatest stencilv1beta1.GetLatestSchemaRequest 40 | var data []byte 41 | 42 | ctx := context.Background() 43 | 44 | if version != 0 { 45 | req.NamespaceId = namespaceID 46 | req.SchemaId = schemaID 47 | req.VersionId = version 48 | res, err := client.GetSchema(ctx, &req) 49 | if err != nil { 50 | return nil, nil, err 51 | } 52 | data = res.GetData() 53 | } else { 54 | reqLatest.NamespaceId = namespaceID 55 | reqLatest.SchemaId = schemaID 56 | res, err := client.GetLatestSchema(ctx, &reqLatest) 57 | if err != nil { 58 | return nil, nil, err 59 | } 60 | data = res.GetData() 61 | } 62 | 63 | reqMeta := stencilv1beta1.GetSchemaMetadataRequest{ 64 | NamespaceId: namespaceID, 65 | SchemaId: schemaID, 66 | } 67 | meta, err := client.GetSchemaMetadata(context.Background(), &reqMeta) 68 | 69 | if err != nil { 70 | return nil, nil, err 71 | } 72 | 73 | return data, meta, nil 74 | } 75 | -------------------------------------------------------------------------------- /clients/java/src/main/java/io/odpf/stencil/StencilClientFactory.java: -------------------------------------------------------------------------------- 1 | package org.raystack.stencil; 2 | 3 | import org.raystack.stencil.cache.SchemaCacheLoader; 4 | import org.raystack.stencil.client.ClassLoadStencilClient; 5 | import org.raystack.stencil.client.MultiURLStencilClient; 6 | import org.raystack.stencil.client.StencilClient; 7 | import org.raystack.stencil.client.URLStencilClient; 8 | import org.raystack.stencil.config.StencilConfig; 9 | import org.raystack.stencil.http.RemoteFileImpl; 10 | import org.raystack.stencil.http.RetryHttpClient; 11 | 12 | import java.util.List; 13 | 14 | 15 | /** 16 | * Provides static methods for the creation of {@link org.raystack.stencil.client.StencilClient} 17 | * object with configurations and various options like 18 | * single URLs, multiple URLs, statsd client for monitoring 19 | * and {@link org.raystack.stencil.SchemaUpdateListener} for callback on schema change. 20 | */ 21 | public class StencilClientFactory { 22 | /** 23 | * @param url URL to fetch and cache protobuf descriptor set in the client 24 | * @param config Stencil configs 25 | * @return Stencil client for single URL 26 | */ 27 | public static StencilClient getClient(String url, StencilConfig config) { 28 | SchemaCacheLoader cacheLoader = new SchemaCacheLoader(new RemoteFileImpl(RetryHttpClient.create(config)), config); 29 | return new URLStencilClient(url, config, cacheLoader); 30 | } 31 | 32 | /** 33 | * @param urls List of URLs to fetch and cache protobuf descriptor sets in the client 34 | * @param config Stencil configs 35 | * @return Stencil client for multiple URLs 36 | */ 37 | public static StencilClient getClient(List urls, StencilConfig config) { 38 | SchemaCacheLoader cacheLoader = new SchemaCacheLoader(new RemoteFileImpl(RetryHttpClient.create(config)), config); 39 | return new MultiURLStencilClient(urls, config, cacheLoader); 40 | } 41 | 42 | /** 43 | * @return Stencil client for getting descriptors from classes in classpath 44 | */ 45 | public static StencilClient getClient() { 46 | return new ClassLoadStencilClient(); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /cmd/graph.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/MakeNowJust/heredoc" 8 | "github.com/raystack/stencil/pkg/graph" 9 | stencilv1beta1 "github.com/raystack/stencil/proto/raystack/stencil/v1beta1" 10 | "github.com/spf13/cobra" 11 | "google.golang.org/protobuf/proto" 12 | "google.golang.org/protobuf/types/descriptorpb" 13 | ) 14 | 15 | func graphSchemaCmd(cdk *CDK) *cobra.Command { 16 | var output, namespaceID string 17 | var version int32 18 | 19 | cmd := &cobra.Command{ 20 | Use: "graph", 21 | Aliases: []string{"g"}, 22 | Short: "View schema dependencies graph", 23 | Args: cobra.ExactArgs(1), 24 | Example: heredoc.Doc(` 25 | $ stencil schema graph booking -n raystack -v 1 -o ./vis.dot 26 | `), 27 | RunE: func(cmd *cobra.Command, args []string) error { 28 | client, cancel, err := createClient(cmd, cdk) 29 | if err != nil { 30 | return err 31 | } 32 | defer cancel() 33 | 34 | schemaID := args[0] 35 | 36 | data, resMetadata, err := fetchSchemaAndMeta(client, version, namespaceID, schemaID) 37 | if err != nil { 38 | return err 39 | } 40 | 41 | format := stencilv1beta1.Schema_Format_name[int32(resMetadata.GetFormat())] 42 | if format != "FORMAT_PROTOBUF" { 43 | fmt.Printf("Graph is not supported for %s", format) 44 | return nil 45 | } 46 | 47 | msg := &descriptorpb.FileDescriptorSet{} 48 | err = proto.Unmarshal(data, msg) 49 | if err != nil { 50 | return fmt.Errorf("invalid file descriptorset file. %w", err) 51 | } 52 | 53 | graph, err := graph.GetProtoFileDependencyGraph(msg) 54 | if err != nil { 55 | return err 56 | } 57 | if err = os.WriteFile(output, []byte(graph.String()), 0666); err != nil { 58 | return err 59 | } 60 | 61 | fmt.Println("Created graph file at", output) 62 | return nil 63 | }, 64 | } 65 | 66 | cmd.Flags().StringVarP(&namespaceID, "namespace", "n", "", "provide namespace/group or entity name") 67 | cmd.MarkFlagRequired("namespace") 68 | 69 | cmd.Flags().Int32VarP(&version, "version", "v", 0, "provide version number") 70 | 71 | cmd.Flags().StringVarP(&output, "output", "o", "./proto_vis.dot", "write to .dot file") 72 | 73 | return cmd 74 | } 75 | -------------------------------------------------------------------------------- /formats/json/schema.go: -------------------------------------------------------------------------------- 1 | package json 2 | 3 | import ( 4 | "github.com/google/uuid" 5 | "github.com/raystack/stencil/core/schema" 6 | "github.com/raystack/stencil/pkg/logger" 7 | "github.com/santhosh-tekuri/jsonschema/v5" 8 | _ "github.com/santhosh-tekuri/jsonschema/v5/httploader" // imported to compile http references in json schema 9 | "go.uber.org/multierr" 10 | ) 11 | 12 | const jsonFormat = "FORMAT_JSON" 13 | const schemaURI = "sample_schema" 14 | 15 | type Schema struct { 16 | data []byte 17 | } 18 | 19 | func (s *Schema) Format() string { 20 | return jsonFormat 21 | } 22 | 23 | func (s *Schema) GetCanonicalValue() *schema.SchemaFile { 24 | id := uuid.NewSHA1(uuid.NameSpaceOID, s.data) 25 | return &schema.SchemaFile{ 26 | ID: id.String(), 27 | Data: s.data, 28 | } 29 | } 30 | 31 | // IsBackwardCompatible checks backward compatibility against given schema 32 | func (s *Schema) IsBackwardCompatible(against schema.ParsedSchema) error { 33 | sc, err := jsonschema.CompileString(schemaURI, string(s.data)) 34 | if err != nil { 35 | logger.Logger.Warn("unable to compile schema to check for backward compatibility") 36 | return err 37 | } 38 | againstSchema, err := jsonschema.CompileString(schemaURI, string(against.GetCanonicalValue().Data)) 39 | if err != nil { 40 | logger.Logger.Warn("unable to compile against schema to check for backward compatibility") 41 | return err 42 | } 43 | jsonSchemaMap := exploreSchema(sc) 44 | againstJsonSchemaMap := exploreSchema(againstSchema) 45 | return compareSchemas(againstJsonSchemaMap, jsonSchemaMap, backwardCompatibility, 46 | []SchemaCompareCheck{CheckPropertyDeleted, TypeCheckExecutor(StandardTypeChecks)}, []SchemaCheck{CheckAdditionalProperties}) 47 | } 48 | 49 | // IsForwardCompatible checks backward compatibility against given schema 50 | func (s *Schema) IsForwardCompatible(against schema.ParsedSchema) error { 51 | return against.IsBackwardCompatible(s) 52 | } 53 | 54 | // IsFullCompatible checks for forward compatibility 55 | func (s *Schema) IsFullCompatible(against schema.ParsedSchema) error { 56 | forwardErr := s.IsForwardCompatible(against) 57 | backwardErr := s.IsBackwardCompatible(against) 58 | return multierr.Combine(forwardErr, backwardErr) 59 | } 60 | -------------------------------------------------------------------------------- /cmd/check.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "os" 8 | 9 | "github.com/MakeNowJust/heredoc" 10 | "github.com/raystack/salt/cli/printer" 11 | stencilv1beta1 "github.com/raystack/stencil/proto/raystack/stencil/v1beta1" 12 | "github.com/spf13/cobra" 13 | "google.golang.org/grpc/status" 14 | ) 15 | 16 | func checkSchemaCmd(cdk *CDK) *cobra.Command { 17 | var comp, file, namespaceID string 18 | var req stencilv1beta1.CheckCompatibilityRequest 19 | 20 | cmd := &cobra.Command{ 21 | Use: "check ", 22 | Args: cobra.ExactArgs(1), 23 | Short: "Check schema compatibility", 24 | Long: heredoc.Doc(` 25 | Check schema compatibility of a local schema 26 | against a remote schema(against) on stencil server.`), 27 | Example: heredoc.Doc(` 28 | $ stencil schema check -n raystack -c COMPATIBILITY_BACKWARD -F ./booking.desc 29 | `), 30 | RunE: func(cmd *cobra.Command, args []string) error { 31 | spinner := printer.Spin("") 32 | defer spinner.Stop() 33 | 34 | fileData, err := os.ReadFile(file) 35 | if err != nil { 36 | return err 37 | } 38 | 39 | client, cancel, err := createClient(cmd, cdk) 40 | if err != nil { 41 | return err 42 | } 43 | defer cancel() 44 | 45 | schemaID := args[0] 46 | 47 | req.Data = fileData 48 | req.NamespaceId = namespaceID 49 | req.SchemaId = schemaID 50 | req.Compatibility = stencilv1beta1.Schema_Compatibility(stencilv1beta1.Schema_Compatibility_value[comp]) 51 | 52 | _, err = client.CheckCompatibility(context.Background(), &req) 53 | if err != nil { 54 | errStatus := status.Convert(err) 55 | return errors.New(errStatus.Message()) 56 | } 57 | 58 | spinner.Stop() 59 | fmt.Printf("\n%s Schema is compatible.\n", printer.Green(printer.Icon("success"))) 60 | return nil 61 | }, 62 | } 63 | 64 | cmd.Flags().StringVarP(&namespaceID, "namespace", "n", "", "Parent namespace ID") 65 | cmd.MarkFlagRequired("namespace") 66 | 67 | cmd.Flags().StringVarP(&comp, "comp", "c", "", "Schema compatibility") 68 | cmd.MarkFlagRequired("comp") 69 | 70 | cmd.Flags().StringVarP(&file, "file", "F", "", "Path to the schema file") 71 | cmd.MarkFlagRequired("file") 72 | 73 | return cmd 74 | } 75 | -------------------------------------------------------------------------------- /formats/protobuf/schema.go: -------------------------------------------------------------------------------- 1 | package protobuf 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/google/uuid" 7 | "github.com/raystack/stencil/core/schema" 8 | "google.golang.org/protobuf/reflect/protoregistry" 9 | ) 10 | 11 | const protobufFormat = "FORMAT_PROTOBUF" 12 | 13 | type Schema struct { 14 | *protoregistry.Files 15 | isValid bool 16 | data []byte 17 | } 18 | 19 | func (s *Schema) Format() string { 20 | return protobufFormat 21 | } 22 | 23 | func (s *Schema) GetCanonicalValue() *schema.SchemaFile { 24 | id := uuid.NewSHA1(uuid.NameSpaceOID, s.data) 25 | return &schema.SchemaFile{ 26 | ID: id.String(), 27 | Types: getAllMessages(s.Files), 28 | Data: s.data, 29 | Fields: getAllFields(s.Files), 30 | } 31 | } 32 | 33 | func (s *Schema) verify(against schema.ParsedSchema) (*Schema, error) { 34 | prev, ok := against.(*Schema) 35 | if against.Format() != protobufFormat && !ok { 36 | return prev, errors.New("different schema formats") 37 | } 38 | return prev, nil 39 | } 40 | 41 | // IsBackwardCompatible checks backward compatibility against given schema 42 | // Allowed changes: field addition 43 | // Disallowed changes: field type change, tag number change, label change, field deletion 44 | func (s *Schema) IsBackwardCompatible(against schema.ParsedSchema) error { 45 | prev, err := s.verify(against) 46 | if err != nil { 47 | return err 48 | } 49 | return compareSchemas(s.Files, prev.Files, backwardCompatibility) 50 | } 51 | 52 | // IsForwardCompatible for protobuf forward compatible is same as backward compatible 53 | // Allowed changes: field addition, field deletion given tag number marked as reserved 54 | // Disallowed changes: field type change, tag number change, label change 55 | func (s *Schema) IsForwardCompatible(against schema.ParsedSchema) error { 56 | prev, err := s.verify(against) 57 | if err != nil { 58 | return err 59 | } 60 | return compareSchemas(s.Files, prev.Files, forwardCompatibility) 61 | } 62 | 63 | // IsFullCompatible for protobuf forward compatible is same as backward compatible 64 | func (s *Schema) IsFullCompatible(against schema.ParsedSchema) error { 65 | prev, err := s.verify(against) 66 | if err != nil { 67 | return err 68 | } 69 | return compareSchemas(s.Files, prev.Files, fullCompatibility) 70 | } 71 | -------------------------------------------------------------------------------- /ui/README.md: -------------------------------------------------------------------------------- 1 | # Getting Started with Create React App 2 | 3 | This project was bootstrapped with [Create React App](https://github.com/facebook/create-react-app). 4 | 5 | ## Available Scripts 6 | 7 | In the project directory, you can run: 8 | 9 | ### `npm start` 10 | 11 | Runs the app in the development mode.\ 12 | Open [http://localhost:3000](http://localhost:3000) to view it in the browser. 13 | 14 | The page will reload if you make edits.\ 15 | You will also see any lint errors in the console. 16 | 17 | ### `npm test` 18 | 19 | Launches the test runner in the interactive watch mode.\ 20 | See the section about [running tests](https://facebook.github.io/create-react-app/docs/running-tests) for more information. 21 | 22 | ### `npm run build` 23 | 24 | Builds the app for production to the `build` folder.\ 25 | It correctly bundles React in production mode and optimizes the build for the best performance. 26 | 27 | The build is minified and the filenames include the hashes.\ 28 | Your app is ready to be deployed! 29 | 30 | See the section about [deployment](https://facebook.github.io/create-react-app/docs/deployment) for more information. 31 | 32 | ### `npm run eject` 33 | 34 | **Note: this is a one-way operation. Once you `eject`, you can’t go back!** 35 | 36 | If you aren’t satisfied with the build tool and configuration choices, you can `eject` at any time. This command will remove the single build dependency from your project. 37 | 38 | Instead, it will copy all the configuration files and the transitive dependencies (webpack, Babel, ESLint, etc) right into your project so you have full control over them. All of the commands except `eject` will still work, but they will point to the copied scripts so you can tweak them. At this point you’re on your own. 39 | 40 | You don’t have to ever use `eject`. The curated feature set is suitable for small and middle deployments, and you shouldn’t feel obligated to use this feature. However we understand that this tool wouldn’t be useful if you couldn’t customize it when you are ready for it. 41 | 42 | ## Learn More 43 | 44 | You can learn more in the [Create React App documentation](https://facebook.github.io/create-react-app/docs/getting-started). 45 | 46 | To learn React, check out the [React documentation](https://reactjs.org/). 47 | -------------------------------------------------------------------------------- /clients/clojure/src/stencil/decode.clj: -------------------------------------------------------------------------------- 1 | (ns stencil.decode 2 | (:require [clojure.string :as string]) 3 | (:import [com.google.protobuf Descriptors$Descriptor Descriptors$FieldDescriptor Descriptors$EnumValueDescriptor DynamicMessage ByteString])) 4 | 5 | (defn- byte-string->bytes 6 | [^ByteString value] 7 | (.toByteArray value)) 8 | 9 | (defn- underscores->hyphen [k] 10 | (string/replace k #"_" "-")) 11 | 12 | (defn- field-name->keyword 13 | [k] 14 | (-> k 15 | underscores->hyphen 16 | keyword)) 17 | 18 | (defn- enum-value->clj-name 19 | [^Descriptors$EnumValueDescriptor value] 20 | (-> value 21 | (.getName) 22 | (field-name->keyword))) 23 | 24 | (defn- fd->keyword 25 | [^Descriptors$FieldDescriptor fd] 26 | (let [name (.getName fd)] 27 | (field-name->keyword name))) 28 | 29 | (declare proto-message->clojure-map) 30 | 31 | (defn- proto-field->clojure-map-fn 32 | [^Descriptors$FieldDescriptor fd] 33 | (let [type-name (-> fd 34 | (.getType) 35 | (.toString)) 36 | transform-fn (case type-name 37 | ("INT32" "UINT32" "SINT32" "FIXED32" "SFIXED32") int 38 | ("INT64" "UINT64" "SINT64" "FIXED64" "SFIXED64") long 39 | "DOUBLE" double 40 | "FLOAT" float 41 | "BOOL" boolean 42 | "STRING" str 43 | "BYTES" byte-string->bytes 44 | "ENUM" enum-value->clj-name 45 | "MESSAGE" proto-message->clojure-map)] 46 | (if (.isRepeated fd) 47 | (partial map transform-fn) 48 | transform-fn))) 49 | 50 | (defn- proto-message->clojure-map 51 | [^DynamicMessage msg] 52 | (let [all-fields (.getAllFields msg) 53 | reducer (fn [acc [k v]] 54 | (->> ((proto-field->clojure-map-fn k) v) 55 | (assoc acc (fd->keyword k))))] 56 | (reduce reducer {} all-fields))) 57 | 58 | (defn- get-dynamic-message 59 | [^Descriptors$Descriptor desc ^"[B" data] 60 | (DynamicMessage/parseFrom desc data)) 61 | 62 | (defn bytes->map 63 | [^Descriptors$Descriptor desc ^"[B" data] 64 | (-> (get-dynamic-message desc data) 65 | proto-message->clojure-map)) 66 | -------------------------------------------------------------------------------- /docs/src/pages/help.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import Layout from '@theme/Layout'; 3 | import useDocusaurusContext from '@docusaurus/useDocusaurusContext'; 4 | import Container from '../core/Container'; 5 | import GridBlock from '../core/GridBlock'; 6 | 7 | export default function Home() { 8 | const { siteConfig } = useDocusaurusContext(); 9 | return ( 10 | 13 |
14 | 15 |

Need help?

16 |

17 | Need a bit of help? We're here for you. Check out our current issues, GitHub discussions, or get support through Slack. 18 |

19 | 26 | The Stencil team has an open source slack workspace to discuss development and support. 27 | Most of the Stencil discussions happen in #stencil channel. 28 |
Join us on Slack 29 | ) 30 | }, 31 | { 32 | title: 'GitHub Issues', 33 | content: ( 34 |
35 | Have a general issue or bug that you've found? We'd love to hear about it in our GitHub issues. This can be feature requests too! 36 |
Go to issues 37 | 38 |
) 39 | }, 40 | { 41 | title: 'GitHub Discussions', 42 | content: ( 43 |
44 | For help and questions about best practices, join our GitHub discussions. Browse and ask questions. 45 |
Go to discussions 46 | 47 |
) 48 | } 49 | ]} 50 | /> 51 | 52 |
53 | 54 | ) 55 | } -------------------------------------------------------------------------------- /cmd/list.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/MakeNowJust/heredoc" 9 | "github.com/raystack/salt/cli/printer" 10 | stencilv1beta1 "github.com/raystack/stencil/proto/raystack/stencil/v1beta1" 11 | "github.com/spf13/cobra" 12 | ) 13 | 14 | func listSchemaCmd(cdk *CDK) *cobra.Command { 15 | var namespace string 16 | var req stencilv1beta1.ListSchemasRequest 17 | 18 | cmd := &cobra.Command{ 19 | Use: "list", 20 | Short: "List all schemas", 21 | Long: heredoc.Doc(` 22 | List schemas in a namespace. 23 | `), 24 | Args: cobra.ExactArgs(0), 25 | Example: heredoc.Doc(` 26 | $ stencil schema list -n raystack 27 | `), 28 | RunE: func(cmd *cobra.Command, args []string) error { 29 | spinner := printer.Spin("") 30 | defer spinner.Stop() 31 | 32 | client, cancel, err := createClient(cmd, cdk) 33 | if err != nil { 34 | return err 35 | } 36 | defer cancel() 37 | 38 | req.Id = namespace 39 | res, err := client.ListSchemas(context.Background(), &req) 40 | if err != nil { 41 | return err 42 | } 43 | schemas := res.GetSchemas() 44 | 45 | // TODO(Ravi): List schemas should also handle namespace not found 46 | if len(schemas) == 0 { 47 | spinner.Stop() 48 | fmt.Printf("No schema found in namespace %s\n", namespace) 49 | return nil 50 | } 51 | 52 | report := [][]string{} 53 | index := 1 54 | report = append(report, []string{ 55 | printer.Bold("INDEX"), 56 | printer.Bold("NAME"), 57 | printer.Bold("FORMAT"), 58 | printer.Bold("COMPATIBILITY"), 59 | printer.Bold("AUTHORITY"), 60 | }) 61 | for _, s := range schemas { 62 | c := s.GetCompatibility().String() 63 | f := s.GetFormat().String() 64 | a := s.GetAuthority() 65 | 66 | if a == "" { 67 | a = "-" 68 | } 69 | report = append(report, []string{printer.Greenf("#%d", index), s.GetName(), dict[f], dict[c], a}) 70 | index++ 71 | } 72 | 73 | spinner.Stop() 74 | fmt.Printf("\nShowing %d of %d schemas in %s\n\n", len(schemas), len(schemas), namespace) 75 | printer.Table(os.Stdout, report) 76 | return nil 77 | }, 78 | } 79 | 80 | cmd.Flags().StringVarP(&namespace, "namespace", "n", "", "Namespace ID") 81 | cmd.MarkFlagRequired("namespace") 82 | 83 | return cmd 84 | } 85 | -------------------------------------------------------------------------------- /docs/docs/glossary.md: -------------------------------------------------------------------------------- 1 | # Glossary 2 | 3 | This section describes the core elements of a schema registry. 4 | 5 | ## Namespace 6 | 7 | A named collection of schemas. Each namespace holds a logically related set of schemas, typically managed by a single entity, belonging to a particular application and/or having a shared access control management scope. Since a schema registry is often a resource with a scope greater than a single application and might even span multiple organizations, it is very useful to put a grouping construct around sets of schemas that are related either by ownership or by a shared subject matter context. A namespace has following attributes: 8 | 9 | - **ID:** Identifies the schema group. 10 | - **Format:** Defines the schema format managed by this namespace. e..g Avro, Protobuf, JSON 11 | - **Compatibility** Schema compatibility constraint type. e.g. Backward, Forward, Full 12 | 13 | ## Schema 14 | 15 | A document describing the structure, names, and types of some structured data payload. Conceptually, a schema is a description of a data structure. Since data structures evolve over time, the schema describing them will also evolve over time. Therefore, a schema often has multiple versions. 16 | 17 | ## Version 18 | 19 | A specific version of a schema document. Even though not prescribed in this specification, an implementation might choose to impose compatibility constraints on versions following the initial version of a schema. 20 | 21 | ## Compatibility 22 | 23 | A key Schema Registry feature is the ability to version schemas as they evolve. Compatibility policies are created at the namespace or schema level, and define evolution rules for each schema. 24 | 25 | After a compatibility policy has been defined for a schema, any subsequent version updates must honor the schema’s original compatibility, to allow for consistent schema evolution. 26 | 27 | Compatibility of schemas can be configured with any of the below values: 28 | 29 | ### Backward 30 | 31 | Indicates that new version of a schema would be compatible with earlier version of that schema. 32 | 33 | ### Forward 34 | 35 | Indicates that an existing schema is compatible with subsequent versions of the schema. 36 | 37 | ### Full 38 | 39 | Indicates that a new version of the schema provides both backward and forward compatibilities. 40 | -------------------------------------------------------------------------------- /clients/java/src/main/java/io/odpf/stencil/cache/SchemaRefreshStrategy.java: -------------------------------------------------------------------------------- 1 | package org.raystack.stencil.cache; 2 | 3 | import java.io.IOException; 4 | import java.util.Map; 5 | import java.util.concurrent.atomic.AtomicInteger; 6 | 7 | import com.google.protobuf.Descriptors; 8 | 9 | import org.json.JSONArray; 10 | import org.json.JSONObject; 11 | 12 | import org.raystack.stencil.DescriptorMapBuilder; 13 | import org.raystack.stencil.exception.StencilRuntimeException; 14 | import org.raystack.stencil.http.RemoteFile; 15 | 16 | public interface SchemaRefreshStrategy { 17 | Map refresh(String url, RemoteFile remoteFile, final Map prevDescriptor); 18 | 19 | static SchemaRefreshStrategy longPollingStrategy() { 20 | return (String url, RemoteFile remoteFile, Map prevDescriptor) -> DescriptorMapBuilder.buildFrom(url, 21 | remoteFile); 22 | } 23 | 24 | static SchemaRefreshStrategy versionBasedRefresh() { 25 | final AtomicInteger lastVersion = new AtomicInteger(); 26 | return (String url, RemoteFile remoteFile, Map prevDescriptor) -> { 27 | try { 28 | byte[] data = remoteFile.fetch(String.format("%s/versions", url)); 29 | JSONObject json = new JSONObject(new String(data)); 30 | JSONArray versions = json.getJSONArray("versions"); 31 | Integer maxVersion = 0; 32 | for (int i = 0; i < versions.length(); i++) { 33 | if (versions.getInt(i) > maxVersion) { 34 | maxVersion = versions.getInt(i); 35 | } 36 | } 37 | if (maxVersion != 0 && maxVersion != lastVersion.get()) { 38 | String newURL = String.format("%s/versions/%d", url, maxVersion); 39 | Map newSchema = DescriptorMapBuilder.buildFrom(newURL, remoteFile); 40 | lastVersion.set(maxVersion); 41 | return newSchema; 42 | } 43 | return prevDescriptor; 44 | } catch (IOException e) { 45 | throw new StencilRuntimeException(e); 46 | } 47 | }; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /clients/java/src/main/java/io/odpf/stencil/client/MultiURLStencilClient.java: -------------------------------------------------------------------------------- 1 | package org.raystack.stencil.client; 2 | 3 | import com.google.protobuf.Descriptors; 4 | import org.raystack.stencil.cache.SchemaCacheLoader; 5 | import org.raystack.stencil.config.StencilConfig; 6 | 7 | import java.io.IOException; 8 | import java.io.Serializable; 9 | import java.util.HashMap; 10 | import java.util.List; 11 | import java.util.Map; 12 | import java.util.Optional; 13 | import java.util.stream.Collectors; 14 | 15 | /** 16 | * {@link StencilClient} implementation that can fetch descriptor sets from multiple URLs 17 | */ 18 | public class MultiURLStencilClient implements Serializable, StencilClient { 19 | 20 | private List stencilClients; 21 | 22 | /** 23 | * @param urls List of URLs to fetch protobuf descriptor sets from 24 | * @param config Stencil configs 25 | * @param cacheLoader Extension of Guava {@link com.google.common.cache.CacheLoader} for Proto Descriptor sets 26 | */ 27 | public MultiURLStencilClient(List urls, StencilConfig config, SchemaCacheLoader cacheLoader) { 28 | stencilClients = urls.stream().map(url -> new URLStencilClient(url, config, cacheLoader)).collect(Collectors.toList()); 29 | } 30 | 31 | @Override 32 | public Descriptors.Descriptor get(String protoClassName) { 33 | Optional requiredStencil = stencilClients.stream().filter(stencilClient -> stencilClient.get(protoClassName) != null).findFirst(); 34 | return requiredStencil.map(stencilClient -> stencilClient.get(protoClassName)).orElse(null); 35 | } 36 | 37 | @Override 38 | public Map getAll() { 39 | Map requiredStencil = new HashMap<>(); 40 | stencilClients.stream().map(StencilClient::getAll) 41 | .forEach(requiredStencil::putAll); 42 | return requiredStencil; 43 | } 44 | 45 | @Override 46 | public void close() { 47 | stencilClients.forEach(c -> { 48 | try { 49 | c.close(); 50 | } catch (IOException e) { 51 | e.printStackTrace(); 52 | } 53 | }); 54 | } 55 | 56 | @Override 57 | public void refresh() { 58 | stencilClients.forEach(c -> { 59 | c.refresh(); 60 | }); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /clients/clojure/test/stencil/testdata/one.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package org.raystack.stencil_clj_test; 4 | 5 | option java_multiple_files = true; 6 | option java_package = "org.raystack.stencil_clj_test"; 7 | 8 | import "google/protobuf/struct.proto"; 9 | import "google/protobuf/timestamp.proto"; 10 | import "google/protobuf/duration.proto"; 11 | import "google/protobuf/wrappers.proto"; 12 | import "google/protobuf/field_mask.proto"; 13 | import "google/protobuf/empty.proto"; 14 | 15 | 16 | enum Group { 17 | UNKNOWN = 0; 18 | VALUE_1 = 1; 19 | VALUE_2 = 2; 20 | VALUE3 = 3; 21 | value4 = 4; 22 | } 23 | // Message with all scalar types 24 | message Scalar { 25 | double field_one = 1; 26 | float float_field = 2; 27 | int32 field_int32 = 3; 28 | int64 field_int64 = 4; 29 | uint32 field_uint32 = 5; 30 | uint64 field_uint64 = 6; 31 | sint32 field_sint32 = 7; 32 | sint64 field_sint64 = 8; 33 | fixed32 field_fixed32 = 9; 34 | fixed64 field_fixed64 = 10; 35 | sfixed32 field_sfixed32 = 11; 36 | sfixed64 field_sfixed64 = 12; 37 | bool field_bool = 13; 38 | string field_string = 14; 39 | bytes field_bytes = 15; 40 | } 41 | 42 | message Wrappers { 43 | google.protobuf.StringValue one = 1; 44 | google.protobuf.DoubleValue two = 2; 45 | google.protobuf.FloatValue three = 3; 46 | google.protobuf.Int64Value four = 4; 47 | google.protobuf.UInt64Value five = 5; 48 | google.protobuf.Int32Value six = 6; 49 | google.protobuf.UInt32Value seven = 7; 50 | google.protobuf.BoolValue eight = 8; 51 | google.protobuf.BytesValue nine = 9; 52 | } 53 | 54 | message SimpleMask { 55 | google.protobuf.FieldMask mask = 1; 56 | } 57 | 58 | message SimpleEmpty { 59 | google.protobuf.Empty empty_field = 1; 60 | } 61 | 62 | message SimpleNested { 63 | Group field_name = 1; 64 | string group = 2; 65 | Scalar nested_field = 3; 66 | google.protobuf.Timestamp duration_field = 4; 67 | google.protobuf.Duration timestamp_field = 5; 68 | } 69 | 70 | message ComplexTypes { 71 | string name = 1; 72 | map map_field = 2; 73 | google.protobuf.Struct struct_field = 3; 74 | } 75 | 76 | message SimpleArray { 77 | repeated Group groups = 1; 78 | repeated string values = 2; 79 | repeated SimpleNested nested_fields = 3; 80 | } 81 | 82 | message Recursive { 83 | string name = 1; 84 | Recursive single_field = 2; 85 | repeated Recursive multi_field = 3; 86 | Group group_field = 4; 87 | } 88 | -------------------------------------------------------------------------------- /.goreleaser.yaml: -------------------------------------------------------------------------------- 1 | project_name: stencil 2 | release: 3 | prerelease: auto 4 | before: 5 | hooks: 6 | - go mod tidy 7 | - make clean 8 | - make ui 9 | builds: 10 | - id: "stencil" 11 | main: ./main.go 12 | binary: stencil 13 | flags: 14 | - -a 15 | ldflags: 16 | - -X github.com/raystack/stencil/config.Version={{.Tag}} 17 | - -X github.com/raystack/stencil/config.BuildCommit={{.FullCommit}} 18 | - -X github.com/raystack/stencil/config.BuildDate={{.Date}} 19 | goos: [darwin, linux, windows] 20 | goarch: [amd64, 386, arm, arm64] 21 | env: 22 | - CGO_ENABLED=0 23 | archives: 24 | - name_template: >- 25 | {{ .ProjectName }}_ 26 | {{- title .Os }}_ 27 | {{- if eq .Arch "amd64" }}x86_64 28 | {{- else if eq .Arch "386" }}i386 29 | {{- else }}{{ .Arch }}{{ end }} 30 | {{- if .Arm }}v{{ .Arm }}{{ end }} 31 | format_overrides: 32 | - goos: windows 33 | format: zip 34 | changelog: 35 | use: "github-native" 36 | sort: asc 37 | filters: 38 | exclude: 39 | - "^docs:" 40 | - "^test:" 41 | - "^build:" 42 | checksum: 43 | name_template: "checksums.txt" 44 | snapshot: 45 | name_template: "{{ .Tag }}-next" 46 | dockers: 47 | - goos: linux 48 | goarch: amd64 49 | ids: 50 | - stencil 51 | dockerfile: Dockerfile 52 | image_templates: 53 | - "docker.io/raystack/{{.ProjectName}}:latest" 54 | - "docker.io/raystack/{{.ProjectName}}:{{ .Version }}" 55 | - "docker.io/raystack/{{.ProjectName}}:{{ .Tag }}-amd64" 56 | nfpms: 57 | - maintainer: Raystack 58 | description: Schema registry 59 | homepage: https://github.com/raystack/stencil 60 | license: Apache 2.0 61 | formats: 62 | - deb 63 | - rpm 64 | scoop: 65 | bucket: 66 | owner: raystack 67 | name: scoop-bucket 68 | homepage: "https://github.com/raystack/stencil" 69 | description: "Schema registry" 70 | license: Apache 2.0 71 | brews: 72 | - name: stencil 73 | homepage: "https://github.com/raystack/stencil" 74 | description: "Schema registry" 75 | tap: 76 | owner: raystack 77 | name: homebrew-tap 78 | license: "Apache 2.0" 79 | folder: Formula 80 | dependencies: 81 | - name: git 82 | install: |- 83 | bin.install "stencil" 84 | commit_author: 85 | name: Ravi Suhag 86 | email: suhag.ravi@gmail.com 87 | -------------------------------------------------------------------------------- /internal/store/postgres/postgres.go: -------------------------------------------------------------------------------- 1 | package postgres 2 | 3 | import ( 4 | "context" 5 | "embed" 6 | "fmt" 7 | "log" 8 | "net/http" 9 | 10 | "github.com/golang-migrate/migrate/v4" 11 | _ "github.com/golang-migrate/migrate/v4/database/postgres" 12 | "github.com/golang-migrate/migrate/v4/source/httpfs" 13 | "github.com/jackc/pgconn" 14 | "github.com/jackc/pgx/v4" 15 | "github.com/jackc/pgx/v4/log/zapadapter" 16 | "github.com/jackc/pgx/v4/pgxpool" 17 | "github.com/pkg/errors" 18 | "github.com/raystack/stencil/internal/store" 19 | "github.com/raystack/stencil/pkg/logger" 20 | ) 21 | 22 | //go:embed migrations 23 | var migrationFs embed.FS 24 | 25 | const ( 26 | resourcePath = "migrations" 27 | ) 28 | 29 | // DB represents postgres database instance 30 | type DB struct { 31 | *pgxpool.Pool 32 | } 33 | 34 | // NewStore create a postgres store 35 | func NewStore(conn string) *DB { 36 | cc, _ := pgxpool.ParseConfig(conn) 37 | cc.ConnConfig.Logger = zapadapter.NewLogger(logger.Logger) 38 | 39 | pgxPool, err := pgxpool.ConnectConfig(context.Background(), cc) 40 | if err != nil { 41 | log.Fatal(err) 42 | } 43 | 44 | return &DB{Pool: pgxPool} 45 | } 46 | 47 | // NewHTTPFSMigrator reads the migrations from httpfs and returns the migrate.Migrate 48 | func NewHTTPFSMigrator(DBConnURL string) (*migrate.Migrate, error) { 49 | src, err := httpfs.New(http.FS(migrationFs), resourcePath) 50 | if err != nil { 51 | return &migrate.Migrate{}, fmt.Errorf("db migrator: %v", err) 52 | } 53 | return migrate.NewWithSourceInstance("httpfs", src, DBConnURL) 54 | } 55 | 56 | // Migrate to run up migrations 57 | func Migrate(connURL string) error { 58 | m, err := NewHTTPFSMigrator(connURL) 59 | if err != nil { 60 | return errors.Wrap(err, "db migrator") 61 | } 62 | defer m.Close() 63 | 64 | if err := m.Up(); err != nil && err != migrate.ErrNoChange { 65 | return errors.Wrap(err, "db migrator") 66 | } 67 | return nil 68 | } 69 | 70 | func wrapError(err error, format string, args ...interface{}) error { 71 | if err == nil { 72 | return err 73 | } 74 | var pgErr *pgconn.PgError 75 | if errors.Is(err, pgx.ErrNoRows) { 76 | return store.NoRowsErr.WithErr(err, fmt.Sprintf(format, args...)) 77 | } 78 | if errors.As(err, &pgErr) { 79 | if pgErr.Code == "23505" { 80 | return store.ConflictErr.WithErr(err, fmt.Sprintf(format, args...)) 81 | } 82 | } 83 | return store.UnknownErr.WithErr(err, fmt.Sprintf(format, args...)) 84 | } 85 | -------------------------------------------------------------------------------- /clients/java/src/main/java/io/odpf/stencil/http/RetryHttpClient.java: -------------------------------------------------------------------------------- 1 | package org.raystack.stencil.http; 2 | 3 | import org.raystack.stencil.config.StencilConfig; 4 | import org.apache.http.HttpResponse; 5 | import org.apache.http.client.ServiceUnavailableRetryStrategy; 6 | import org.apache.http.client.config.RequestConfig; 7 | import org.apache.http.impl.client.CloseableHttpClient; 8 | import org.apache.http.impl.client.HttpClientBuilder; 9 | import org.apache.http.protocol.HttpContext; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | 13 | public class RetryHttpClient { 14 | private static final Logger logger = LoggerFactory.getLogger(RemoteFileImpl.class); 15 | 16 | 17 | public static CloseableHttpClient create(StencilConfig stencilConfig) { 18 | 19 | int timeout = stencilConfig.getFetchTimeoutMs(); 20 | long backoffMs = stencilConfig.getFetchBackoffMinMs(); 21 | int retries = stencilConfig.getFetchRetries(); 22 | 23 | logger.info("initialising HTTP client with timeout: {}ms, backoff: {}ms, max retry attempts: {}", timeout, backoffMs, retries); 24 | 25 | RequestConfig requestConfig = RequestConfig.custom() 26 | .setConnectTimeout(timeout) 27 | .setSocketTimeout(timeout).build(); 28 | 29 | return HttpClientBuilder.create() 30 | .setDefaultRequestConfig(requestConfig) 31 | .setDefaultHeaders(stencilConfig.getFetchHeaders()) 32 | .setConnectionManagerShared(true) 33 | .setServiceUnavailableRetryStrategy(new ServiceUnavailableRetryStrategy() { 34 | long waitPeriod = backoffMs; 35 | 36 | @Override 37 | public boolean retryRequest(HttpResponse response, int executionCount, HttpContext context) { 38 | if (executionCount <= retries && response.getStatusLine().getStatusCode() >= 400) { 39 | logger.info("Retrying requests, attempts left: {}", retries - executionCount); 40 | waitPeriod *= 2; 41 | return true; 42 | } 43 | return false; 44 | } 45 | 46 | @Override 47 | public long getRetryInterval() { 48 | return waitPeriod; 49 | } 50 | }) 51 | .build(); 52 | } 53 | 54 | } 55 | --------------------------------------------------------------------------------