├── .dockerignore ├── .github └── workflows │ ├── maven_macos_12.yml │ ├── maven_macos_13.yml │ ├── maven_macos_latest.yml │ ├── maven_ubuntu_latest.yml │ ├── maven_windows_2019.yml │ ├── maven_windows_2022.yml │ └── maven_windows_latest.yml ├── .gitignore ├── .travis.yml ├── .vscode └── settings.json ├── Dockerfile ├── LICENSE ├── LOCALDEV.md ├── README.md ├── agent └── opentelemetry-javaagent.jar ├── data └── grafana-data │ └── datasources │ └── datasource.yml ├── diffy.env ├── docker-compose-dependencies.yml ├── docker-compose.yml ├── etc ├── loki-local.yaml ├── prometheus.yaml ├── promtail-local.yaml ├── tempo-local.yaml └── tempo-query.yaml ├── example ├── dependency-reduced-pom.xml ├── downstream.sh ├── pom.xml ├── run.sh ├── src │ └── main │ │ └── java │ │ └── com │ │ └── sn126 │ │ └── example │ │ └── HttpLambdaServer.java └── traffic.sh ├── frontend ├── .gitignore ├── index.html ├── package.json ├── public │ └── logo.jpeg ├── src │ ├── App.tsx │ ├── app │ │ ├── hooks.ts │ │ └── store.ts │ ├── features │ │ ├── appbar │ │ │ └── AppBarView.tsx │ │ ├── differences │ │ │ ├── DifferenceResults.ts │ │ │ ├── DifferencesQueryArgs.ts │ │ │ └── DifferencesView.tsx │ │ ├── endpoints │ │ │ ├── EndpointMeta.ts │ │ │ └── EndpointsView.tsx │ │ ├── fields │ │ │ ├── Endpoint.ts │ │ │ ├── EndpointFieldPrefix.ts │ │ │ ├── FieldNode.ts │ │ │ ├── FieldsQueryArgs.ts │ │ │ ├── FieldsView.tsx │ │ │ └── Metric.ts │ │ ├── info │ │ │ ├── InfoView.tsx │ │ │ └── infoApiSlice.ts │ │ ├── noise │ │ │ └── noiseApiSlice.ts │ │ ├── overrides │ │ │ ├── OverrideView.tsx │ │ │ ├── editor │ │ │ │ └── CodeEditor.tsx │ │ │ ├── overrideSlice.ts │ │ │ └── transformationsApiSlice.ts │ │ ├── requests │ │ │ ├── Request.ts │ │ │ ├── RequestView.tsx │ │ │ └── requestsApiSlice.ts │ │ └── selections │ │ │ └── selectionsSlice.ts │ ├── main.tsx │ └── vite-env.d.ts ├── tsconfig.json ├── tsconfig.node.json └── vite.config.ts ├── images ├── diffy-ui.png ├── diffy_topology.png ├── graphana_prometheus.png ├── jaeger_diffy_response_traces.png ├── loki_diffy.png ├── loki_to_tempo.png ├── prometheus_diffy.png ├── prometheus_schema_metrics.png └── tempo_diffy.png ├── localdev.properties ├── maven_docker_cache.xml ├── pom.xml └── src ├── main ├── resources │ ├── META-INF │ │ └── spring.factories │ └── application.yml └── scala │ └── ai │ └── diffy │ ├── ApiController.scala │ ├── Main.java │ ├── NoiseController.scala │ ├── ObjectMapperCustomizer.scala │ ├── Renderer.scala │ ├── Settings.scala │ ├── analysis │ ├── DifferenceAnalyzer.scala │ ├── DifferenceCounter.scala │ ├── DifferenceResult.java │ ├── DynamicAnalyzer.scala │ ├── Field.scala │ ├── FieldDifference.java │ ├── InMemoryDifferenceCollector.scala │ ├── JoinedDifferences.scala │ ├── Report.scala │ └── Responses.java │ ├── compare │ └── Difference.scala │ ├── flat │ ├── FlatDifference.scala │ ├── FlatIndexedCollection.scala │ └── FlatObject.scala │ ├── functional │ ├── algebra │ │ ├── Bijection.java │ │ ├── UnsafeFunction.java │ │ ├── monoids │ │ │ ├── functions │ │ │ │ ├── BinaryOperator.java │ │ │ │ ├── DecaOperator.java │ │ │ │ ├── HexaOperator.java │ │ │ │ ├── NonaOperator.java │ │ │ │ ├── NullOperator.java │ │ │ │ ├── OctaOperator.java │ │ │ │ ├── PentaOperator.java │ │ │ │ ├── QuadOperator.java │ │ │ │ ├── SeptaOperator.java │ │ │ │ ├── SymmetricUnaryOperator.java │ │ │ │ ├── TernaryOperator.java │ │ │ │ └── UnaryOperator.java │ │ │ └── suppliers │ │ │ │ ├── BiSupplierOperator.java │ │ │ │ ├── NullSupplierOperator.java │ │ │ │ ├── QuadSupplierOperator.java │ │ │ │ ├── TriSupplierOperator.java │ │ │ │ └── UnarySupplierOperator.java │ │ └── unions │ │ │ ├── SFFBinaryOperator.java │ │ │ └── SFSBinaryOperator.java │ ├── endpoints │ │ ├── BiDependentEndpoint.java │ │ ├── DependentEndpoint.java │ │ ├── Endpoint.java │ │ ├── HexaDependentEndpoint.java │ │ ├── IndependentEndpoint.java │ │ ├── OctaDependentEndpoint.java │ │ ├── PentaDependentEndpoint.java │ │ ├── QuadDependentEndpoint.java │ │ ├── SeptaDependentEndpoint.java │ │ ├── SymmetricDependentEndpoint.java │ │ └── TriDependentEndpoint.java │ ├── functions │ │ ├── DecaFunction.java │ │ ├── HexaFunction.java │ │ ├── NonaFunction.java │ │ ├── OctaFunction.java │ │ ├── PentaFunction.java │ │ ├── Proceeder.java │ │ ├── QuadFunction.java │ │ ├── SeptaFunction.java │ │ ├── TriFunction.java │ │ └── Try.java │ └── topology │ │ ├── Async.java │ │ ├── AsyncCommonPoolUnaryOperator.java │ │ ├── AsyncUnaryOperator.java │ │ ├── ControlFlowLogger.java │ │ ├── InvocationLogger.java │ │ ├── SpanWrapper.java │ │ └── TriConsumer.java │ ├── interpreter │ ├── Lambda.java │ ├── Transformer.java │ └── http │ │ ├── HttpLambdaServer.java │ │ ├── candidate.js │ │ └── master.js │ ├── lifter │ ├── AnalysisRequest.scala │ ├── FieldMap.scala │ ├── HtmlLifter.scala │ ├── HttpLifter.scala │ ├── JsonLifter.scala │ ├── Message.scala │ └── StringLifter.scala │ ├── metrics │ └── MetricsReceiver.scala │ ├── proxy │ ├── AnalysisSpanLogger.java │ ├── HttpEndpoint.java │ ├── HttpMessage.java │ ├── HttpRequest.java │ ├── HttpResponse.java │ ├── MulticastProxy.java │ └── ReactorHttpDifferenceProxy.java │ ├── repository │ ├── DifferenceResultRepository.java │ ├── Noise.java │ └── NoiseRepository.java │ ├── transformations │ ├── Transformation.java │ ├── TransformationCachingService.java │ ├── TransformationController.java │ ├── TransformationEdge.java │ └── TransformationRepository.java │ └── util │ ├── Future.java │ ├── Memoize.scala │ ├── ResourceMatcher.scala │ └── ResponseMode.java └── test ├── resources ├── application.yml ├── echo.js ├── lambda.js └── payload.json └── scala └── ai └── diffy ├── HttpHeaderValuesTest.java ├── IntegrationTest.java ├── LoadGenerator.java ├── compare └── DifferenceTest.java ├── functional ├── algebra │ └── BijectionTest.java └── topology │ └── InvocationLoggerTest.java ├── interpreter ├── SimpleTransformerTest.java ├── TransformerTest.java └── transform.js └── proxy └── ReactorHttpDifferenceProxyTest.java /.dockerignore: -------------------------------------------------------------------------------- 1 | target/ 2 | dist/ 3 | project/boot/ 4 | project/plugins/project/ 5 | project/plugins/src_managed/ 6 | *.log 7 | *.tmproj 8 | lib_managed/ 9 | *.swp 10 | *.iml 11 | .idea/ 12 | .idea/* 13 | .DS_Store 14 | .ensime 15 | .ivyjars 16 | .vscode/ 17 | .metals/ 18 | example/ai/diffy/examples/http/ 19 | data/logs/ 20 | data/loki-data/ 21 | data/tempo-data/ 22 | 23 | # dependencies 24 | frontend/node_modules 25 | frontend/.pnp 26 | frontend/.pnp.js 27 | 28 | # testing 29 | frontend/coverage 30 | 31 | # production 32 | frontend/build 33 | frontend/node 34 | 35 | # misc 36 | frontend/.DS_Store 37 | frontend/.env.local 38 | frontend/.env.development.local 39 | frontend/.env.test.local 40 | frontend/.env.production.local 41 | 42 | frontend/npm-debug.log* 43 | frontend/yarn-debug.log* 44 | frontend/yarn-error.log* 45 | 46 | frontend/package-lock.json 47 | frontend/yarn.lock -------------------------------------------------------------------------------- /.github/workflows/maven_macos_12.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven 3 | 4 | name: MacOS (12) 5 | env: 6 | CI: false 7 | 8 | on: 9 | push: 10 | branches: [ "master" ] 11 | pull_request: 12 | branches: [ "master" ] 13 | 14 | jobs: 15 | build: 16 | 17 | runs-on: macos-12 18 | 19 | steps: 20 | - uses: actions/checkout@v3 21 | - name: Set up JDK 17 22 | uses: actions/setup-java@v3 23 | with: 24 | java-version: '17' 25 | distribution: 'temurin' 26 | cache: maven 27 | - name: Build with Maven 28 | run: mvn -B package --file pom.xml 29 | -------------------------------------------------------------------------------- /.github/workflows/maven_macos_13.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven 3 | 4 | name: MacOS (13) 5 | env: 6 | CI: false 7 | 8 | on: 9 | push: 10 | branches: [ "master" ] 11 | pull_request: 12 | branches: [ "master" ] 13 | 14 | jobs: 15 | build: 16 | 17 | runs-on: macos-13 18 | 19 | steps: 20 | - uses: actions/checkout@v3 21 | - name: Set up JDK 17 22 | uses: actions/setup-java@v3 23 | with: 24 | java-version: '17' 25 | distribution: 'temurin' 26 | cache: maven 27 | - name: Build with Maven 28 | run: mvn -B package --file pom.xml 29 | -------------------------------------------------------------------------------- /.github/workflows/maven_macos_latest.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven 3 | 4 | name: MacOS (latest) 5 | env: 6 | CI: false 7 | 8 | on: 9 | push: 10 | branches: [ "master" ] 11 | pull_request: 12 | branches: [ "master" ] 13 | 14 | jobs: 15 | build: 16 | 17 | runs-on: macos-latest 18 | 19 | steps: 20 | - uses: actions/checkout@v3 21 | - name: Set up JDK 17 22 | uses: actions/setup-java@v3 23 | with: 24 | java-version: '17' 25 | distribution: 'temurin' 26 | cache: maven 27 | - name: Build with Maven 28 | run: mvn -B package --file pom.xml 29 | -------------------------------------------------------------------------------- /.github/workflows/maven_ubuntu_latest.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven 3 | 4 | name: Ubuntu (latest) 5 | env: 6 | CI: false 7 | 8 | on: 9 | push: 10 | branches: [ "master" ] 11 | pull_request: 12 | branches: [ "master" ] 13 | 14 | jobs: 15 | build: 16 | 17 | runs-on: ubuntu-latest 18 | 19 | steps: 20 | - uses: actions/checkout@v3 21 | - name: Set up JDK 17 22 | uses: actions/setup-java@v3 23 | with: 24 | java-version: '17' 25 | distribution: 'temurin' 26 | cache: maven 27 | - name: Build with Maven 28 | run: mvn -B package --file pom.xml 29 | -------------------------------------------------------------------------------- /.github/workflows/maven_windows_2019.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven 3 | 4 | name: Windows (2019) 5 | env: 6 | CI: false 7 | 8 | on: 9 | push: 10 | branches: [ "master" ] 11 | pull_request: 12 | branches: [ "master" ] 13 | 14 | jobs: 15 | build: 16 | 17 | runs-on: windows-2019 18 | 19 | steps: 20 | - uses: actions/checkout@v3 21 | - name: Set up JDK 17 22 | uses: actions/setup-java@v3 23 | with: 24 | java-version: '17' 25 | distribution: 'temurin' 26 | cache: maven 27 | - name: Build with Maven 28 | run: mvn -B package --file pom.xml 29 | -------------------------------------------------------------------------------- /.github/workflows/maven_windows_2022.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven 3 | 4 | name: Windows (2022) 5 | env: 6 | CI: false 7 | 8 | on: 9 | push: 10 | branches: [ "master" ] 11 | pull_request: 12 | branches: [ "master" ] 13 | 14 | jobs: 15 | build: 16 | 17 | runs-on: windows-2022 18 | 19 | steps: 20 | - uses: actions/checkout@v3 21 | - name: Set up JDK 17 22 | uses: actions/setup-java@v3 23 | with: 24 | java-version: '17' 25 | distribution: 'temurin' 26 | cache: maven 27 | - name: Build with Maven 28 | run: mvn -B package --file pom.xml 29 | -------------------------------------------------------------------------------- /.github/workflows/maven_windows_latest.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven 3 | 4 | name: Windows (latest) 5 | env: 6 | CI: false 7 | 8 | on: 9 | push: 10 | branches: [ "master" ] 11 | pull_request: 12 | branches: [ "master" ] 13 | 14 | jobs: 15 | build: 16 | 17 | runs-on: windows-latest 18 | 19 | steps: 20 | - uses: actions/checkout@v3 21 | - name: Set up JDK 17 22 | uses: actions/setup-java@v3 23 | with: 24 | java-version: '17' 25 | distribution: 'temurin' 26 | cache: maven 27 | - name: Build with Maven 28 | run: mvn -B package --file pom.xml 29 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | dist/ 3 | project/boot/ 4 | project/plugins/project/ 5 | project/plugins/src_managed/ 6 | *.log 7 | *.tmproj 8 | lib_managed/ 9 | *.swp 10 | *.iml 11 | .idea/ 12 | .idea/* 13 | .DS_Store 14 | .ensime 15 | .ivyjars 16 | .vscode/ 17 | .metals/ 18 | example/ai/diffy/examples/http/ 19 | data/logs/ 20 | data/loki-data/ 21 | data/tempo-data/ 22 | 23 | # dependencies 24 | frontend/node_modules 25 | frontend/.pnp 26 | frontend/.pnp.js 27 | 28 | # testing 29 | frontend/coverage 30 | 31 | # production 32 | frontend/dist 33 | frontend/node 34 | 35 | # misc 36 | frontend/.vscode 37 | 38 | frontend/npm-debug.log* 39 | frontend/yarn-debug.log* 40 | frontend/yarn-error.log* 41 | 42 | frontend/package-lock.json 43 | frontend/yarn.lock -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | os: 3 | - linux 4 | - osx 5 | - windows 6 | script: mvn package -DskipTests 7 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "files.watcherExclude": { 3 | "**/target": true 4 | } 5 | } -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # build image 2 | FROM maven:3.8-eclipse-temurin-17-focal AS builder 3 | ENV HOME=/usr/local/src 4 | RUN mkdir -p $HOME 5 | WORKDIR $HOME 6 | # install cached version of pom.xml 7 | ADD maven_docker_cache.xml $HOME 8 | RUN mvn verify -f maven_docker_cache.xml --fail-never 9 | 10 | # install node v16.14.0 and yarn v1.22.19 11 | RUN mvn com.github.eirslett:frontend-maven-plugin:install-node-and-yarn -DnodeVersion=v16.14.0 -DyarnVersion=v1.22.19 -f maven_docker_cache.xml 12 | 13 | # install dependencies in frontend/package.json 14 | RUN mkdir -p $HOME/frontend 15 | ADD frontend/package.json $HOME/frontend 16 | RUN mvn com.github.eirslett:frontend-maven-plugin:yarn -f maven_docker_cache.xml 17 | 18 | # install dependencies in pom.xml 19 | ADD pom.xml $HOME 20 | RUN mvn verify --fail-never 21 | 22 | # finally, copy, compile, bundle, and package everything 23 | ADD . $HOME 24 | RUN mvn package 25 | RUN ls 26 | RUN mv target /target 27 | RUN mv agent /agent 28 | 29 | # production image 30 | FROM maven:3.8-eclipse-temurin-17-focal 31 | COPY --from=builder /target/diffy.jar /diffy.jar 32 | COPY --from=builder /agent/opentelemetry-javaagent.jar /opentelemetry-javaagent.jar 33 | ENTRYPOINT ["java", "-javaagent:opentelemetry-javaagent.jar", "-jar", "diffy.jar"] 34 | CMD [] -------------------------------------------------------------------------------- /LOCALDEV.md: -------------------------------------------------------------------------------- 1 | # Local Development 2 | ## Downstream dependencies 3 | We use docker compose to standup all downstream dependencies required for our [Advanced Deployment Configurations](/ADVANCED.md) 4 | with the following command : 5 | ```docker compose -f localdev/docker-compose-dependecies.yml up``` 6 | 7 | ## Running Diffy with localdev dependecies 8 | We also use the following command line argument to point to corresponding configuration require to run Diffy: 9 | ```-Dspring.config.location=localdev.yml``` 10 | -------------------------------------------------------------------------------- /agent/opentelemetry-javaagent.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendiffy/diffy/6c8235b1aeb2b246b759d32a5db95a91f665a909/agent/opentelemetry-javaagent.jar -------------------------------------------------------------------------------- /data/grafana-data/datasources/datasource.yml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | deleteDatasources: 4 | - name: Prometheus 5 | - name: Tempo 6 | - name: Loki 7 | 8 | datasources: 9 | - name: Prometheus 10 | type: prometheus 11 | access: proxy 12 | orgId: 1 13 | url: http://prometheus:9090 14 | basicAuth: false 15 | isDefault: false 16 | version: 1 17 | editable: false 18 | - name: Tempo 19 | type: tempo 20 | access: proxy 21 | orgId: 1 22 | url: http://tempo-query:16686 23 | basicAuth: false 24 | isDefault: false 25 | version: 1 26 | editable: false 27 | apiVersion: 1 28 | uid: tempo 29 | - name: Tempo-Multitenant 30 | type: tempo 31 | access: proxy 32 | orgId: 1 33 | url: http://tempo-query:16686 34 | basicAuth: false 35 | isDefault: false 36 | version: 1 37 | editable: false 38 | apiVersion: 1 39 | uid: tempo-authed 40 | jsonData: 41 | httpHeaderName1: 'Authorization' 42 | secureJsonData: 43 | httpHeaderValue1: 'Bearer foo-bar-baz' 44 | 45 | - name: Loki 46 | type: loki 47 | access: proxy 48 | orgId: 1 49 | url: http://loki:3100 50 | basicAuth: false 51 | isDefault: false 52 | version: 1 53 | editable: false 54 | apiVersion: 1 55 | jsonData: 56 | derivedFields: 57 | - datasourceUid: tempo 58 | matcherRegex: \[.+,(.+),.+\] 59 | name: TraceID 60 | url: $${__value.raw} -------------------------------------------------------------------------------- /diffy.env: -------------------------------------------------------------------------------- 1 | logging.pattern.console=%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(%5p) %clr([diffy,%X{trace_id},%X{span_id}]) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m %n%wEx 2 | logging.pattern.file=%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(%5p) %clr([diffy,%X{trace_id},%X{span_id}]) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m %n%wEx 3 | -------------------------------------------------------------------------------- /docker-compose-dependencies.yml: -------------------------------------------------------------------------------- 1 | version: "3.8" 2 | 3 | services: 4 | mongodb: 5 | image: mongo 6 | container_name: mongodb 7 | environment: 8 | - MONGO_INITDB_ROOT_USERNAME=root 9 | - MONGO_INITDB_ROOT_PASSWORD=pass12345 10 | ports: 11 | - 27017:27017 12 | healthcheck: 13 | test: echo 'db.stats().ok' | mongo localhost:27017/test --quiet 14 | interval: 10s 15 | timeout: 10s 16 | retries: 10 17 | restart: unless-stopped 18 | 19 | loki: 20 | image: grafana/loki:2.2.0 21 | command: -config.file=/etc/loki/loki-local.yaml 22 | user: "0" 23 | ports: 24 | - "3101:3100" # loki needs to be exposed so it receives logs 25 | environment: 26 | - JAEGER_AGENT_HOST=tempo 27 | - JAEGER_ENDPOINT=http://tempo:14268/api/traces # send traces to Tempo 28 | - JAEGER_SAMPLER_TYPE=const 29 | - JAEGER_SAMPLER_PARAM=1 30 | volumes: 31 | - ./etc/loki-local.yaml:/etc/loki/loki-local.yaml 32 | - ./data/loki-data:/tmp/loki 33 | 34 | tempo: 35 | image: grafana/tempo:0.7.0 36 | command: ["-config.file=/etc/tempo.yaml"] 37 | volumes: 38 | - ./etc/tempo-local.yaml:/etc/tempo.yaml 39 | - ./data/tempo-data:/tmp/tempo 40 | restart: unless-stopped 41 | ports: 42 | - "14268:14268" # jaeger ingest, Jaeger - Thrift HTTP 43 | - "14250:14250" # Jaeger - GRPC 44 | - "55680:55680" # OpenTelemetry 45 | - "3102:3100" # tempo 46 | # - "4317:4317" # returns not implemented 47 | 48 | tempo-query: 49 | image: grafana/tempo-query:0.7.0 50 | command: ["--grpc-storage-plugin.configuration-file=/etc/tempo-query.yaml"] 51 | volumes: 52 | - ./etc/tempo-query.yaml:/etc/tempo-query.yaml 53 | ports: 54 | - "16686:16686" # jaeger-ui 55 | depends_on: 56 | - tempo 57 | 58 | primary: 59 | ports: 60 | - "9100:5000" 61 | image: diffy/example-service:production 62 | 63 | secondary: 64 | ports: 65 | - "9200:5000" 66 | image: diffy/example-service:production 67 | 68 | candidate: 69 | ports: 70 | - "9000:5000" 71 | image: diffy/example-service:candidate 72 | 73 | promtail: 74 | image: grafana/promtail:2.2.0 75 | command: -config.file=/etc/promtail/promtail-local.yaml 76 | volumes: 77 | - ./etc/promtail-local.yaml:/etc/promtail/promtail-local.yaml 78 | - ./data/logs:/app/logs 79 | depends_on: 80 | - loki 81 | 82 | prometheus: 83 | image: prom/prometheus:latest 84 | volumes: 85 | - ./etc/prometheus.yaml:/etc/prometheus.yaml 86 | entrypoint: 87 | - /bin/prometheus 88 | - --config.file=/etc/prometheus.yaml 89 | ports: 90 | - "9090:9090" 91 | 92 | grafana: 93 | image: grafana/grafana:7.4.0-ubuntu 94 | volumes: 95 | - ./data/grafana-data/datasources:/etc/grafana/provisioning/datasources 96 | - ./data/grafana-data/dashboards-provisioning:/etc/grafana/provisioning/dashboards 97 | - ./data/grafana-data/dashboards:/var/lib/grafana/dashboards 98 | environment: 99 | - GF_AUTH_ANONYMOUS_ENABLED=true 100 | - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin 101 | - GF_AUTH_DISABLE_LOGIN_FORM=true 102 | ports: 103 | - "3000:3000" 104 | depends_on: 105 | - prometheus 106 | - tempo-query 107 | - loki -------------------------------------------------------------------------------- /etc/loki-local.yaml: -------------------------------------------------------------------------------- 1 | auth_enabled: false 2 | 3 | server: 4 | http_listen_port: 3100 5 | 6 | ingester: 7 | lifecycler: 8 | address: 127.0.0.1 9 | ring: 10 | kvstore: 11 | store: inmemory 12 | replication_factor: 1 13 | final_sleep: 0s 14 | chunk_idle_period: 1h # Any chunk not receiving new logs in this time will be flushed 15 | max_chunk_age: 1h # All chunks will be flushed when they hit this age, default is 1h 16 | chunk_target_size: 1048576 # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first 17 | chunk_retain_period: 30s # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m) 18 | max_transfer_retries: 0 # Chunk transfers disabled 19 | 20 | 21 | schema_config: 22 | configs: 23 | - from: 2020-10-24 24 | store: boltdb-shipper 25 | object_store: filesystem 26 | schema: v11 27 | index: 28 | prefix: index_ 29 | period: 24h 30 | 31 | storage_config: 32 | boltdb_shipper: 33 | active_index_directory: /tmp/loki/boltdb-shipper-active 34 | cache_location: /tmp/loki/boltdb-shipper-cache 35 | cache_ttl: 24h # Can be increased for faster performance over longer query periods, uses more disk space 36 | shared_store: filesystem 37 | filesystem: 38 | directory: /tmp/loki/chunks 39 | 40 | compactor: 41 | working_directory: /tmp/loki/boltdb-shipper-compactor 42 | shared_store: filesystem 43 | 44 | limits_config: 45 | reject_old_samples: true 46 | reject_old_samples_max_age: 168h 47 | 48 | chunk_store_config: 49 | max_look_back_period: 0s 50 | 51 | table_manager: 52 | retention_deletes_enabled: false 53 | retention_period: 0s 54 | 55 | ruler: 56 | storage: 57 | type: local 58 | local: 59 | directory: /tmp/loki/rules 60 | rule_path: /tmp/loki/rules-temp 61 | ring: 62 | kvstore: 63 | store: inmemory 64 | enable_api: true -------------------------------------------------------------------------------- /etc/prometheus.yaml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 15s 3 | evaluation_interval: 15s 4 | 5 | scrape_configs: 6 | - job_name: 'prometheus' 7 | static_configs: 8 | - targets: ['localhost:9090'] 9 | 10 | - job_name: 'tempo' 11 | static_configs: 12 | - targets: ['tempo:3100'] 13 | 14 | - job_name: 'loki' 15 | static_configs: 16 | - targets: ['loki:3100'] 17 | 18 | - job_name: 'diffy' 19 | metrics_path: /actuator/prometheus 20 | static_configs: 21 | - targets: ['host.docker.internal:8888'] 22 | -------------------------------------------------------------------------------- /etc/promtail-local.yaml: -------------------------------------------------------------------------------- 1 | server: 2 | http_listen_port: 3101 3 | 4 | clients: 5 | - url: http://loki:3100/loki/api/v1/push 6 | 7 | positions: 8 | filename: /tmp/positions.yaml 9 | 10 | target_config: 11 | sync_period: 10s 12 | 13 | scrape_configs: 14 | - job_name: diffy 15 | static_configs: 16 | - targets: 17 | - localhost 18 | labels: 19 | job: diffy 20 | __path__: /app/logs/diffy*.log 21 | pipeline_stages: 22 | - match: 23 | selector: '{job="diffy"}' 24 | stages: 25 | - regex: 26 | expression: '^(?P\d{4}-\d{2}-\d{2}\s\d{1,2}\:\d{2}\:\d{2}\.\d{3})\s+(?P[A-Z]{4,5})\s[(?P.*),(?P.*),(?P.*)]\s(?P\d)\s---\s[\s*(?P.*)]\s(?P.*)\s+\:\s(?P.*)$' 27 | - labels: 28 | timestamp: 29 | level: 30 | serviceName: 31 | traceId: 32 | spanId: 33 | pid: 34 | thread: 35 | logger: 36 | message: 37 | - timestamp: 38 | format: '2006-01-02 15:04:05.000' 39 | source: timestamp 40 | # https://grafana.com/docs/loki/latest/clients/promtail/stages/multiline/ 41 | - multiline: 42 | firstline: '^\d{4}-\d{2}-\d{2}\s\d{1,2}\:\d{2}\:\d{2}\.\d{3}' 43 | max_wait_time: 3s 44 | -------------------------------------------------------------------------------- /etc/tempo-local.yaml: -------------------------------------------------------------------------------- 1 | auth_enabled: false 2 | 3 | server: 4 | http_listen_port: 3100 5 | 6 | distributor: 7 | receivers: # this configuration will listen on all ports and protocols that tempo is capable of. 8 | jaeger: # the receives all come from the OpenTelemetry collector. more configuration information can 9 | protocols: # be found there: https://github.com/open-telemetry/opentelemetry-collector/tree/master/receiver 10 | thrift_http: # 11 | grpc: # for a production deployment you should only enable the receivers you need! 12 | thrift_binary: 13 | thrift_compact: 14 | zipkin: 15 | otlp: 16 | protocols: 17 | http: 18 | grpc: 19 | opencensus: 20 | 21 | ingester: 22 | trace_idle_period: 10s # the length of time after a trace has not received spans to consider it complete and flush it 23 | #traces_per_block: 1_000_000 24 | max_block_duration: 5m # this much time passes 25 | 26 | compactor: 27 | compaction: 28 | compaction_window: 1h # blocks in this time window will be compacted together 29 | max_compaction_objects: 1000000 # maximum size of compacted blocks 30 | block_retention: 1h 31 | compacted_block_retention: 10m 32 | 33 | storage: 34 | trace: 35 | backend: local # backend configuration to use 36 | wal: 37 | path: /tmp/tempo/wal # where to store the the wal locally 38 | #bloom_filter_false_positive: .05 # bloom filter false positive rate. lower values create larger filters but fewer false positives 39 | #index_downsample: 10 # number of traces per index record 40 | local: 41 | path: /tmp/tempo/blocks 42 | pool: 43 | max_workers: 100 # the worker pool mainly drives querying, but is also used for polling the blocklist 44 | queue_depth: 10000 -------------------------------------------------------------------------------- /etc/tempo-query.yaml: -------------------------------------------------------------------------------- 1 | backend: "tempo:3100" -------------------------------------------------------------------------------- /example/dependency-reduced-pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | com.sn126 5 | example 6 | 1.0.0-SNAPSHOT 7 | 8 | ${project.artifactId} 9 | 10 | 11 | maven-compiler-plugin 12 | 3.8.1 13 | 14 | 18 15 | 18 16 | UTF-8 17 | 18 | 19 | 20 | maven-shade-plugin 21 | 3.4.1 22 | 23 | 24 | package 25 | 26 | shade 27 | 28 | 29 | 30 | 31 | 32 | maven-jar-plugin 33 | 3.1.0 34 | 35 | 36 | 37 | com.sn126.example.HttpLambdaServer 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | io.projectreactor 48 | reactor-bom 49 | 2022.0.2 50 | pom 51 | import 52 | 53 | 54 | 55 | 56 | -------------------------------------------------------------------------------- /example/downstream.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | echo "Build primary, secondary, and candidate servers" && \ 3 | mvn package -f example/pom.xml && \ 4 | 5 | echo "Deploy primary, secondary, and candidate servers" && \ 6 | java -jar example/target/example.jar 9100 9200 9000 7 | -------------------------------------------------------------------------------- /example/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | com.sn126 8 | example 9 | 1.0.0-SNAPSHOT 10 | 11 | 12 | 13 | 14 | io.projectreactor 15 | reactor-bom 16 | 2022.0.2 17 | pom 18 | import 19 | 20 | 21 | 22 | 23 | 24 | io.projectreactor.netty 25 | reactor-netty-core 26 | 27 | 28 | io.projectreactor.netty 29 | reactor-netty-http 30 | 31 | 32 | 33 | io.netty 34 | netty-resolver-dns-native-macos 35 | 4.1.82.Final 36 | osx-aarch_64 37 | 38 | 39 | org.reactivestreams 40 | reactive-streams 41 | 1.0.4 42 | 43 | 44 | ch.qos.logback 45 | logback-classic 46 | 1.2.6 47 | 48 | 49 | 50 | 51 | ${project.artifactId} 52 | 53 | 54 | org.apache.maven.plugins 55 | maven-compiler-plugin 56 | 3.8.1 57 | 58 | 18 59 | 18 60 | UTF-8 61 | 62 | 63 | 64 | org.apache.maven.plugins 65 | maven-shade-plugin 66 | 3.4.1 67 | 68 | 69 | package 70 | 71 | shade 72 | 73 | 74 | 75 | 76 | 77 | 78 | org.apache.maven.plugins 79 | maven-jar-plugin 80 | 3.1.0 81 | 82 | 83 | 84 | com.sn126.example.HttpLambdaServer 85 | 86 | 87 | 88 | 89 | 90 | 91 | -------------------------------------------------------------------------------- /example/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ "$1" = "start" ]; 4 | then 5 | echo "Your Diffy UI will be reached at http://localhost:8888" && \ 6 | echo "You can run \"example/downstream.sh\" to send traffic to your Diffy instance." && \ 7 | echo "Building Diffy" && \ 8 | mvn package && \ 9 | 10 | echo "Deploy Diffy" && \ 11 | java -jar ./target/diffy.jar \ 12 | --candidate='localhost:9000' \ 13 | --master.primary='localhost:9100' \ 14 | --master.secondary='localhost:9200' \ 15 | --allowHttpSideEffects='true' \ 16 | --responseMode='candidate' \ 17 | --service.protocol='http' \ 18 | --serviceName='ExampleService' \ 19 | --proxy.port=8880 \ 20 | --http.port=8888 21 | else 22 | echo "Please make sure you run \"example/downstream.sh\" before running \"example/run.sh start\"" 23 | fi 24 | -------------------------------------------------------------------------------- /example/src/main/java/com/sn126/example/HttpLambdaServer.java: -------------------------------------------------------------------------------- 1 | package com.sn126.example; 2 | 3 | import io.netty.handler.codec.http.HttpResponseStatus; 4 | import reactor.core.publisher.Mono; 5 | import reactor.netty.DisposableServer; 6 | import reactor.netty.http.server.HttpServer; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | import java.io.File; 10 | import java.io.IOException; 11 | import java.util.function.Function; 12 | 13 | public class HttpLambdaServer { 14 | private final Logger log = LoggerFactory.getLogger(HttpLambdaServer.class); 15 | 16 | private final DisposableServer server; 17 | 18 | public HttpLambdaServer(int port, Function lambda) { 19 | server = HttpServer.create() 20 | .port(port) 21 | .handle((req, res) -> { 22 | log.info("Received traffic on port {}", port); 23 | return Mono.fromFuture( 24 | req.receive().aggregate().asString().toFuture().thenApply(lambda) 25 | ).flatMap(responseBody -> 26 | res.header("UPPERCASE_TO_LOWERCASE", "must_convert") 27 | .sendString(Mono.justOrEmpty(responseBody)) 28 | .then() 29 | ); 30 | }).bindNow(); 31 | } 32 | 33 | public static void main(String[] args) throws Exception { 34 | HttpLambdaServer primary = new HttpLambdaServer(Integer.parseInt(args[0]), s->s); 35 | HttpLambdaServer secondary = new HttpLambdaServer(Integer.parseInt(args[1]), s->s); 36 | HttpLambdaServer candidate = new HttpLambdaServer(Integer.parseInt(args[2]), s->s.toUpperCase()); 37 | 38 | primary.server.onDispose().block(); 39 | secondary.server.onDispose().block(); 40 | candidate.server.onDispose().block(); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /example/traffic.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | echo "Sending some traffic to your Diffy instance" 3 | for i in {1..1} 4 | do 5 | sleep 0.1 6 | curl -s -i -X POST -d 'Microsoft' -H "Content-Type:application/text;Canonical-Resource:text" http://localhost:8880/text > /dev/null 7 | sleep 0.1 8 | curl -s -i -X POST -d '{"name":"Microsoft"}' -H "Content-Type:application/json;Canonical-Resource:json" http://localhost:8880/json > /dev/null 9 | sleep 0.1 10 | curl -s -i -X POST -d 'Twitter' -H "Content-Type:application/text;Canonical-Resource:text" http://localhost:8880/text > /dev/null 11 | sleep 0.1 12 | curl -s -i -X POST -d '{"name":"Twitter"}' -H "Content-Type:application/json;Canonical-Resource:json" http://localhost:8880/json > /dev/null 13 | sleep 0.1 14 | curl -s -i -X POST -d 'Airbnb' -H "Content-Type:application/text;Canonical-Resource:text" http://localhost:8880/text > /dev/null 15 | sleep 0.1 16 | curl -s -i -X POST -d '{"name":"Airbnb"}' -H "Content-Type:application/json;Canonical-Resource:json" http://localhost:8880/json > /dev/null 17 | sleep 0.1 18 | curl -s -i -X POST -d 'Mixpanel' -H "Content-Type:application/text;Canonical-Resource:text" http://localhost:8880/text > /dev/null 19 | sleep 0.1 20 | curl -s -i -X POST -d '{"name":"Mixpanel"}' -H "Content-Type:application/json;Canonical-Resource:json" http://localhost:8880/json > /dev/null 21 | sleep 0.1 22 | curl -s -i -X POST -d 'Cigna' -H "Content-Type:application/text;Canonical-Resource:text" http://localhost:8880/text > /dev/null 23 | sleep 0.1 24 | curl -s -i -X POST -d '{"name":"Cigna"}' -H "Content-Type:application/json;Canonical-Resource:json" http://localhost:8880/json > /dev/null 25 | done 26 | -------------------------------------------------------------------------------- /frontend/.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | pnpm-debug.log* 8 | lerna-debug.log* 9 | 10 | node_modules 11 | dist 12 | dist-ssr 13 | *.local 14 | 15 | # Editor directories and files 16 | .vscode/* 17 | !.vscode/extensions.json 18 | .idea 19 | .DS_Store 20 | *.suo 21 | *.ntvs* 22 | *.njsproj 23 | *.sln 24 | *.sw? 25 | -------------------------------------------------------------------------------- /frontend/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Diffy 8 | 9 | 10 |
11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /frontend/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "frontend-redux", 3 | "private": true, 4 | "version": "0.0.0", 5 | "proxy": "http://localhost:8888", 6 | "scripts": { 7 | "start": "react-scripts start", 8 | "dev": "vite", 9 | "build": "tsc && vite build", 10 | "preview": "vite preview" 11 | }, 12 | "dependencies": { 13 | "@emotion/react": "^11.10.4", 14 | "@emotion/styled": "^11.10.4", 15 | "@mui/icons-material": "^5.10.9", 16 | "@mui/lab": "^5.0.0-alpha.104", 17 | "@mui/material": "^5.10.10", 18 | "@mui/x-tree-view": "^7.12.0", 19 | "@reduxjs/toolkit": "^1.8.6", 20 | "@types/lodash": "^4.14.186", 21 | "@wojtekmaj/react-datetimerange-picker": "^5.2.0", 22 | "ace-builds": "^1.12.5", 23 | "bootstrap": "^5.2.2", 24 | "d3-drag": "^3.0.0", 25 | "d3-force": "^3.0.0", 26 | "d3-format": "^3.1.0", 27 | "d3-selection": "^3.0.0", 28 | "lodash": "^4.17.21", 29 | "react": "^18.2.0", 30 | "react-ace": "^10.1.0", 31 | "react-dom": "^18.2.0", 32 | "react-redux": "^8.0.4" 33 | }, 34 | "devDependencies": { 35 | "@types/react": "^18.0.17", 36 | "@types/react-dom": "^18.0.6", 37 | "@vitejs/plugin-react": "^2.1.0", 38 | "typescript": "^4.6.4", 39 | "vite": "^3.1.0" 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /frontend/public/logo.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendiffy/diffy/6c8235b1aeb2b246b759d32a5db95a91f665a909/frontend/public/logo.jpeg -------------------------------------------------------------------------------- /frontend/src/App.tsx: -------------------------------------------------------------------------------- 1 | import { Dialog, DialogContent, Grid } from '@mui/material'; 2 | import 'bootstrap/dist/css/bootstrap.min.css'; 3 | 4 | import { useAppDispatch, useAppSelector } from './app/hooks' 5 | import InfoView from './features/info/InfoView'; 6 | import EndpointsView from './features/endpoints/EndpointsView'; 7 | import AppBarView from "./features/appbar/AppBarView"; 8 | import { FieldsView } from './features/fields/FieldsView'; 9 | import { RequestView } from './features/requests/RequestView'; 10 | import { closeInfoView, closeRequestView } from './features/selections/selectionsSlice'; 11 | import { closeOverrideView } from './features/overrides/overrideSlice'; 12 | import { DifferencesView } from './features/differences/DifferencesView'; 13 | import { OverrideView } from './features/overrides/OverrideView'; 14 | 15 | function App() { 16 | const dispatch = useAppDispatch(); 17 | const infoIsOpen = useAppSelector((state) => state.selections.infoIsOpen); 18 | const requestIsOpen = useAppSelector((state) => state.selections.requestIsOpen); 19 | const overrideViewIsOpen = useAppSelector((state) => state.overrides.overrideViewIsOpen); 20 | 21 | return ( 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | {dispatch(closeInfoView())}}> 41 | 42 | 43 | 44 | 45 | 46 | {dispatch(closeRequestView())}}> 51 | 52 | 53 | 54 | 55 | 56 | {dispatch(closeOverrideView())}}> 61 | 62 | 63 | 64 | 65 | 66 | 67 | ) 68 | } 69 | 70 | export default App 71 | -------------------------------------------------------------------------------- /frontend/src/app/hooks.ts: -------------------------------------------------------------------------------- 1 | import { TypedUseSelectorHook, useDispatch, useSelector } from "react-redux"; 2 | import { RootState, AppDispatch } from "./store"; 3 | 4 | export const useAppDispatch = () => useDispatch(); 5 | export const useAppSelector: TypedUseSelectorHook= useSelector; -------------------------------------------------------------------------------- /frontend/src/app/store.ts: -------------------------------------------------------------------------------- 1 | import { configureStore } from "@reduxjs/toolkit"; 2 | import selectionsReducer from '../features/selections/selectionsSlice'; 3 | import overridesReducer from '../features/overrides/overrideSlice'; 4 | import { apiInfoSlice } from "../features/info/infoApiSlice"; 5 | import { apiRequestsSlice } from "../features/requests/requestsApiSlice"; 6 | import { apiNoiseSlice } from "../features/noise/noiseApiSlice"; 7 | import { apiTransformationsSlice } from "../features/overrides/transformationsApiSlice"; 8 | 9 | export const store = configureStore({ 10 | reducer: { 11 | selections: selectionsReducer, 12 | overrides: overridesReducer, 13 | [apiInfoSlice.reducerPath]: apiInfoSlice.reducer, 14 | [apiRequestsSlice.reducerPath]: apiRequestsSlice.reducer, 15 | [apiNoiseSlice.reducerPath]: apiNoiseSlice.reducer, 16 | [apiTransformationsSlice.reducerPath]: apiTransformationsSlice.reducer 17 | }, 18 | middleware: (getDefaultMiddleware) => { 19 | return getDefaultMiddleware() 20 | .concat(apiInfoSlice.middleware) 21 | .concat(apiRequestsSlice.middleware) 22 | .concat(apiNoiseSlice.middleware) 23 | .concat(apiTransformationsSlice.middleware); 24 | } 25 | }); 26 | 27 | export type AppDispatch = typeof store.dispatch; 28 | export type RootState = ReturnType; -------------------------------------------------------------------------------- /frontend/src/features/appbar/AppBarView.tsx: -------------------------------------------------------------------------------- 1 | import { AppBar, Toolbar, Typography, Tooltip, IconButton, Switch, Divider, Link } from "@mui/material" 2 | import SettingsIcon from '@mui/icons-material/Settings'; 3 | import NotesIcon from '@mui/icons-material/Notes'; 4 | import AnalyticsIcon from '@mui/icons-material/Analytics'; 5 | import AccountTreeIcon from '@mui/icons-material/AccountTree'; 6 | import JavascriptIcon from '@mui/icons-material/Javascript'; 7 | 8 | import DateTimeRangePicker from '@wojtekmaj/react-datetimerange-picker'; 9 | import '@wojtekmaj/react-datetimerange-picker/dist/DateTimeRangePicker.css'; 10 | import 'react-calendar/dist/Calendar.css'; 11 | import 'react-clock/dist/Clock.css'; 12 | 13 | import { fetchinfo } from "../info/infoApiSlice"; 14 | import { useAppDispatch, useAppSelector } from '../../app/hooks' 15 | import { openInfoView, setDateTimeRange, toggleNoiseCancellation } from '../selections/selectionsSlice'; 16 | import { openOverrideView } from '../overrides/overrideSlice'; 17 | 18 | export default function AppBarView(){ 19 | const info = fetchinfo(); 20 | const excludeNoise = useAppSelector((state) => state.selections.noiseCancellationIsOn); 21 | const {start, end} = useAppSelector((state) => state.selections.dateTimeRange); 22 | const dispatch = useAppDispatch(); 23 | return 24 | 25 | {info.name} 26 | { 28 | const [s, e] = (Array.isArray(range) && range[0] && range[1]) ? [range[0], range[1]] : [new Date(Date.now() - 24*3600*1000), new Date()]; 29 | dispatch(setDateTimeRange({start: s.getTime(), end: e.getTime()})); 30 | }} 31 | value={[new Date(start), new Date(end)]} 32 | disableClock={true} 33 | /> 34 | 35 | 36 | {dispatch(toggleNoiseCancellation())}} 40 | /> 41 | 42 | 43 | 44 | dispatch(openOverrideView())}> 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | {dispatch(openInfoView())}}> 73 | 74 | 75 | 76 | 77 | 78 | } -------------------------------------------------------------------------------- /frontend/src/features/differences/DifferenceResults.ts: -------------------------------------------------------------------------------- 1 | export interface DifferenceResults { 2 | endpoint: string; 3 | path: string; 4 | requests: Request[]; 5 | } 6 | interface Request { 7 | id: string; 8 | differences: Map; 9 | } 10 | interface Difference { 11 | type: string; 12 | left: any; 13 | right: any; 14 | } 15 | -------------------------------------------------------------------------------- /frontend/src/features/differences/DifferencesQueryArgs.ts: -------------------------------------------------------------------------------- 1 | export interface DifferencesQueryArgs { 2 | selectedEndpoint: string; 3 | selectedFieldPrefix: string; 4 | excludeNoise: boolean; 5 | includeWeights: boolean; 6 | start: number; 7 | end: number; 8 | } 9 | -------------------------------------------------------------------------------- /frontend/src/features/differences/DifferencesView.tsx: -------------------------------------------------------------------------------- 1 | import {IconButton, keyframes, List, ListItem, ListItemText, ListSubheader, Table, TableBody,TableCell, TableHead, TableRow } from '@mui/material'; 2 | import OpenInNewIcon from '@mui/icons-material/OpenInNew'; 3 | 4 | import { useAppDispatch, useAppSelector } from '../../app/hooks'; 5 | import { useFetchDifferencesQuery } from '../noise/noiseApiSlice'; 6 | import { DifferenceResults } from './DifferenceResults'; 7 | import { openRequestView, selectRequest } from '../selections/selectionsSlice'; 8 | 9 | export function DifferencesView(){ 10 | const dispatch = useAppDispatch(); 11 | const excludeNoise = useAppSelector((state) => state.selections.noiseCancellationIsOn); 12 | const selectedEndpoint = useAppSelector((state) => state.selections.endpointName) || 'unknown'; 13 | const selectedFieldPrefix = useAppSelector((state) => state.selections.fieldPrefix) || 'unknown'; 14 | const {start, end} = useAppSelector((state) => state.selections.dateTimeRange); 15 | const differenceResults = useFetchDifferencesQuery({excludeNoise, selectedEndpoint, selectedFieldPrefix, includeWeights:true, start, end}).data || {endpoint:'undefined', path:'undefined', requests:[]}; 16 | if(!differenceResults.requests.length) { 17 | return Differences}> 18 | No differences. 19 | 20 | } 21 | const {requests} = (differenceResults as DifferenceResults); 22 | const differences = requests.flatMap((request) => { 23 | const requestId = request.id 24 | return Object.entries(request.differences).flatMap(([key, diff]) => { 25 | if(!key.startsWith(selectedFieldPrefix)){ 26 | return []; 27 | } 28 | const {type, left, right} = diff 29 | return [{requestId, type, left, right, key:`${selectedEndpoint}.${key}.${requestId}`}]; 30 | }); 31 | }); 32 | return Differences}> 33 | 34 | 35 | 36 | {'Type'} 37 | {'Expected'} 38 | {'Actual'} 39 | {'Details'} 40 | 41 | 42 | 43 | { 44 | differences.map(({requestId, type, left, right, key}) => { 45 | return 46 | {type} 47 | {left} 48 | {right} 49 | 50 | { 52 | dispatch(selectRequest(requestId)); 53 | dispatch(openRequestView()); 54 | }}> 55 | 56 | 57 | 58 | ; 59 | })} 60 | 61 |
62 | ; 63 | } -------------------------------------------------------------------------------- /frontend/src/features/endpoints/EndpointMeta.ts: -------------------------------------------------------------------------------- 1 | export interface EndpointMeta { 2 | total: number; 3 | differences: number; 4 | } 5 | -------------------------------------------------------------------------------- /frontend/src/features/endpoints/EndpointsView.tsx: -------------------------------------------------------------------------------- 1 | import { List, ListItem, ListItemButton, ListItemText, ListSubheader } from '@mui/material'; 2 | 3 | import { useAppDispatch, useAppSelector } from '../../app/hooks' 4 | import { EndpointMeta } from "./EndpointMeta"; 5 | import { selectEndpoint } from '../selections/selectionsSlice'; 6 | import { useFetchNoiseQuery, useFetchEndpointsQuery } from '../noise/noiseApiSlice'; 7 | 8 | export default function EndpointsView() { 9 | const dispatch = useAppDispatch(); 10 | const excludeNoise = useAppSelector((state) => state.selections.noiseCancellationIsOn); 11 | const selectedEndpoint = useAppSelector((state) => state.selections.endpointName) || 'undefined'; 12 | const {start, end} = useAppSelector((state) => state.selections.dateTimeRange); 13 | const noisyFields = useFetchNoiseQuery(selectedEndpoint).data || []; 14 | const endpoints: Map = useFetchEndpointsQuery({excludeNoise, start, end}).data || new Map(); 15 | const entries = Object.entries(endpoints); 16 | return (Endpoints}> 17 | {!entries.length?No endpoints.{excludeNoise && noisyFields.length?'(Some ignored)':''}: 18 | entries.map(([name, {total, differences}]) => { 19 | return {dispatch(selectEndpoint(name))}}> 20 | 21 | 22 | })} 23 | ); 24 | } -------------------------------------------------------------------------------- /frontend/src/features/fields/Endpoint.ts: -------------------------------------------------------------------------------- 1 | import { Metric } from "./Metric"; 2 | 3 | export interface Endpoint { 4 | fields: Map; 5 | } 6 | -------------------------------------------------------------------------------- /frontend/src/features/fields/EndpointFieldPrefix.ts: -------------------------------------------------------------------------------- 1 | export interface EndpointFieldPrefix { 2 | endpoint: string; 3 | fieldPrefix: string; 4 | } 5 | -------------------------------------------------------------------------------- /frontend/src/features/fields/FieldNode.ts: -------------------------------------------------------------------------------- 1 | import { NamedMetric, isNamedMetric } from "./Metric"; 2 | 3 | export interface FieldNode { 4 | id: string, 5 | name: string, 6 | children: FieldNode[], 7 | namedMetric: NamedMetric|undefined 8 | } 9 | 10 | export function nodeOf(obj: any, path: string, name: string): FieldNode { 11 | const id = path?`${path}.${name}` : name; 12 | const children = childrenOf(obj, id); 13 | const namedMetric = children.length ? undefined : obj as NamedMetric; 14 | return { 15 | id, 16 | name, 17 | children, 18 | namedMetric 19 | } 20 | } 21 | 22 | function childrenOf(obj: any, objName: string): FieldNode[] { 23 | if(typeof obj !== 'object' || isNamedMetric(obj)) { 24 | return []; 25 | } 26 | return Object.keys(obj).map(name => nodeOf(obj[name], objName, name)); 27 | } -------------------------------------------------------------------------------- /frontend/src/features/fields/FieldsQueryArgs.ts: -------------------------------------------------------------------------------- 1 | export interface FieldsQueryArgs { 2 | selectedEndpoint: string; 3 | excludeNoise: boolean; 4 | includeWeights: boolean; 5 | start: number; 6 | end: number; 7 | } 8 | -------------------------------------------------------------------------------- /frontend/src/features/fields/Metric.ts: -------------------------------------------------------------------------------- 1 | export interface Metric { 2 | noise: number; 3 | relative_difference: number; 4 | absolute_difference: number; 5 | differences: number; 6 | weight: number; 7 | } 8 | 9 | const zero: NamedMetric = { 10 | name: 'NoDifference', 11 | noise: 0, 12 | relative_difference: 0, 13 | absolute_difference: 0, 14 | differences: 0, 15 | weight: 0 16 | } 17 | 18 | const metricKeys: Set = new Set(Object.keys(zero)) 19 | 20 | export type NamedMetric = Metric & {name: string} 21 | export function isNamedMetric(obj: any): obj is NamedMetric { 22 | if(!obj){ 23 | return false; 24 | } 25 | const objKeys = Object.keys(obj) 26 | return metricKeys.size === objKeys.length && objKeys.map(x => metricKeys.has(x)).reduce((acc, x) => acc && x); 27 | } -------------------------------------------------------------------------------- /frontend/src/features/info/InfoView.tsx: -------------------------------------------------------------------------------- 1 | import DeleteIcon from '@mui/icons-material/Delete'; 2 | import { Alert, Grid, IconButton, Snackbar, Tooltip, Typography } from '@mui/material'; 3 | import { useAppDispatch, useAppSelector } from '../../app/hooks' 4 | 5 | import { fetchinfo, useDeleteRequestsMutation } from './infoApiSlice'; 6 | import { openDeleteRequestsAlert, closeDeleteRequestsAlert } from '../selections/selectionsSlice'; 7 | 8 | export default function InfoView(){ 9 | const dispatch = useAppDispatch(); 10 | const info = fetchinfo(); 11 | const alertIsOpen = useAppSelector((state) => state.selections.deleteRequestAlertIsOpen); 12 | const closeAlert = () => dispatch(closeDeleteRequestsAlert()); 13 | const [deleteRequests, { isLoading: isUpdating, isSuccess }] = useDeleteRequestsMutation(); 14 | 15 | return 16 | 17 | Candidate Server 18 | 19 | 20 | {info.candidate.target} 21 | 22 | 23 | Primary Server 24 | 25 | 26 | {info.primary.target} 27 | 28 | 29 | Secondary Server 30 | 31 | 32 | {info.secondary.target} 33 | 34 | 35 | Protocol 36 | 37 | 38 | {info.protocol} 39 | 40 | 41 | Last Reset 42 | 43 | 44 | {new Date(info.last_reset).toDateString()} 45 | 46 | 47 | Thresholds 48 | 49 | 50 | {info.relativeThreshold}% relative, {info.absoluteThreshold}% absolute 51 | 52 | 53 | 54 | deleteRequests().then(() => dispatch(openDeleteRequestsAlert()))}> 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | All requests deleted! 66 | 67 | 68 | 69 | 70 | } -------------------------------------------------------------------------------- /frontend/src/features/info/infoApiSlice.ts: -------------------------------------------------------------------------------- 1 | import { createApi, fetchBaseQuery } from "@reduxjs/toolkit/query/react"; 2 | 3 | export interface Info { 4 | name: string, 5 | primary: Target, 6 | secondary: Target, 7 | candidate: Target, 8 | relativeThreshold: number, 9 | last_reset: number, 10 | absoluteThreshold: number, 11 | protocol: string, 12 | } 13 | export interface Target { 14 | target: String 15 | } 16 | export const apiInfoSlice = createApi({ 17 | reducerPath: 'info', 18 | baseQuery: fetchBaseQuery({ 19 | baseUrl: '/api/1' 20 | }), 21 | endpoints(builder) { 22 | return { 23 | fetchInfo: builder.query({ 24 | query(){ 25 | return '/info'; 26 | } 27 | }), 28 | deleteRequests: builder.mutation({ 29 | query(){ 30 | return { 31 | url: `/clear`, 32 | method: 'GET' 33 | } 34 | }, 35 | }) 36 | } 37 | }, 38 | }); 39 | 40 | export const {useFetchInfoQuery, useDeleteRequestsMutation} = apiInfoSlice; 41 | export function fetchinfo(){ 42 | const target = 'Unknown'; 43 | return useFetchInfoQuery().data || { 44 | name: 'Unknown', 45 | primary: {target}, 46 | secondary: {target}, 47 | candidate: {target}, 48 | relativeThreshold: 20, 49 | last_reset: 0, 50 | absoluteThreshold: 0.03, 51 | protocol: "http", 52 | } 53 | } -------------------------------------------------------------------------------- /frontend/src/features/noise/noiseApiSlice.ts: -------------------------------------------------------------------------------- 1 | import { createApi, fetchBaseQuery } from "@reduxjs/toolkit/query/react"; 2 | import { DifferenceResults } from "../differences/DifferenceResults"; 3 | import { DifferencesQueryArgs } from "../differences/DifferencesQueryArgs"; 4 | 5 | import { EndpointMeta } from "../endpoints/EndpointMeta"; 6 | import { Endpoint } from "../fields/Endpoint"; 7 | import { EndpointFieldPrefix } from "../fields/EndpointFieldPrefix"; 8 | import { FieldsQueryArgs } from "../fields/FieldsQueryArgs"; 9 | 10 | export const apiNoiseSlice = createApi({ 11 | reducerPath: 'noiseApi', 12 | tagTypes: ['Noise'], 13 | baseQuery: fetchBaseQuery({ 14 | baseUrl: '/api/1', 15 | }), 16 | endpoints(builder) { 17 | return { 18 | fetchNoise: builder.query({ 19 | query(endpointName){ 20 | return `/noise/${endpointName}`; 21 | }, 22 | providesTags: ['Noise'] 23 | }), 24 | postNoise: builder.mutation({ 25 | query({endpoint, fieldPrefix, isNoise}){ 26 | return { 27 | url: `/noise/${endpoint}/prefix/${fieldPrefix}`, 28 | method: 'POST', 29 | headers: { 30 | 'Content-Type': 'application/json' 31 | }, 32 | body: JSON.stringify({isNoise}) 33 | } 34 | }, 35 | invalidatesTags: ['Noise'], 36 | }), 37 | fetchEndpoints: builder.query, {excludeNoise:boolean, start:number, end:number}>({ 38 | query({excludeNoise, start, end}){ 39 | return `/endpoints?exclude_noise=${excludeNoise}&start=${start}&end=${end}`; 40 | }, 41 | providesTags: ['Noise'] 42 | }), 43 | fetchFields: builder.query({ 44 | query({selectedEndpoint, includeWeights, excludeNoise, start, end}){ 45 | return `/endpoints/${selectedEndpoint}/stats?include_weights=${includeWeights}&exclude_noise=${excludeNoise}&start=${start}&end=${end}`; 46 | }, 47 | providesTags: ['Noise'] 48 | }), 49 | fetchDifferences: builder.query({ 50 | query(args){ 51 | return `/endpoints/${args.selectedEndpoint}/fields/${args.selectedFieldPrefix}/results?include_weights=${args.includeWeights}&exclude_noise=${args.excludeNoise}&start=${args.start}&end=${args.end}`; 52 | }, 53 | providesTags: ['Noise'] 54 | }) 55 | } 56 | }, 57 | }); 58 | 59 | export const { 60 | useFetchNoiseQuery, 61 | usePostNoiseMutation, 62 | useFetchEndpointsQuery, 63 | useFetchFieldsQuery, 64 | useFetchDifferencesQuery 65 | } = apiNoiseSlice; -------------------------------------------------------------------------------- /frontend/src/features/overrides/OverrideView.tsx: -------------------------------------------------------------------------------- 1 | import { Grid, Tooltip, Typography } from '@mui/material'; 2 | 3 | import { useAppDispatch, useAppSelector } from '../../app/hooks' 4 | 5 | import { highlightEdge, selectEdge} from './overrideSlice'; 6 | import CodeEditor from './editor/CodeEditor'; 7 | 8 | export function OverrideView(){ 9 | const dispatch = useAppDispatch(); 10 | const highlightedEdge = useAppSelector((state) => state.overrides.highlightedEdge); 11 | const selectedEdge = useAppSelector((state) => state.overrides.selectedEdge); 12 | 13 | function isHighlighted(edge: string): boolean { 14 | return highlightedEdge === edge || highlightedEdge === 'all' || isSelected(edge); 15 | } 16 | function isSelected(edge: string): boolean { 17 | return selectedEdge === edge || selectedEdge === 'all'; 18 | } 19 | 20 | const radius = 20 21 | const delta = {x: 100, y:100} 22 | type Coord = {x: number,y:number} 23 | type Edge = {start:Coord, end:Coord} 24 | const all = {start:{x:0,y:200}, end:{x:delta.x, y:200}} 25 | const candidate = {start: all.end, end: {x: all.end.x+delta.x, y: all.end.y-delta.y}} 26 | const primary = {start: all.end, end: {x: all.end.x+delta.x, y: all.end.y}} 27 | const secondary = {start: all.end, end: {x: all.end.x+delta.x, y: all.end.y+delta.y}} 28 | 29 | function drawEdge(name: string, edge: Edge) { 30 | return <>{dispatch(highlightEdge(name))}} 33 | onMouseOut={()=>{dispatch(highlightEdge(undefined))}} 34 | onClick={() => dispatch(selectEdge(name))} 35 | > 36 | 42 | 43 | {dispatch(highlightEdge(name))}} 46 | onMouseOut={()=>{dispatch(highlightEdge(undefined))}} 47 | onClick={() => dispatch(selectEdge(name))} 48 | > 49 | 53 | 54 | {name}; 58 | } 59 | return 60 | 61 | Request Pattern 62 | 63 | {drawEdge('candidate', candidate)} 64 | {drawEdge('primary', primary)} 65 | {drawEdge('secondary', secondary)} 66 | {drawEdge('all', all)} 67 | 68 | 69 | 70 | {!selectedEdge ? <> : } 71 | 72 | 73 | } -------------------------------------------------------------------------------- /frontend/src/features/overrides/editor/CodeEditor.tsx: -------------------------------------------------------------------------------- 1 | import SaveIcon from '@mui/icons-material/Save'; 2 | import SettingsBackupRestoreIcon from '@mui/icons-material/SettingsBackupRestore'; 3 | import { Alert, Grid, IconButton, Snackbar, Tooltip, Typography } from '@mui/material'; 4 | 5 | import AceEditor from "react-ace"; 6 | import "ace-builds/src-noconflict/mode-java"; 7 | import "ace-builds/src-noconflict/theme-github"; 8 | import "ace-builds/src-noconflict/ext-language_tools" 9 | 10 | import { useAppDispatch, useAppSelector } from '../../../app/hooks' 11 | import { openAlert, closeAlert} from '../overrideSlice'; 12 | import { setTransformationJs} from '../overrideSlice'; 13 | import {useFetchOverrideQuery, useUpdateOverrideMutation} from '../transformationsApiSlice'; 14 | 15 | export default function CodeEditor() { 16 | const dispatch = useAppDispatch(); 17 | const selectedEdge = useAppSelector((state) => state.overrides.selectedEdge) || 'none'; 18 | const alertIsOpen = useAppSelector((state) => state.overrides.alertIsOpen); 19 | const close = () => dispatch(closeAlert()); 20 | const [updateOverride] = useUpdateOverrideMutation(); 21 | const currentTxJs = useAppSelector((state) => state.overrides.currentTransformationJs); 22 | const {data} = useFetchOverrideQuery(selectedEdge); 23 | const remoteTx = !!data ? data : {injectionPoint:'none', transformationJs:'rt => rt'}; 24 | const txJs = !!currentTxJs ? currentTxJs : remoteTx.transformationJs; 25 | return 26 | Overriding traffic to {selectedEdge} with the following function: 27 | 28 | selectedEdge && updateOverride({injectionPoint: selectedEdge, transformationJs: currentTxJs || '(request)=>(request)'}) 33 | .then(() => { 34 | dispatch(setTransformationJs(undefined)); 35 | dispatch(openAlert(`Save success. Transformation will be applied to ${selectedEdge}.`)); 36 | }) 37 | }> 38 | 39 | 40 | 41 | 42 | 43 | selectedEdge && updateOverride({injectionPoint: selectedEdge, transformationJs: '(request)=>(request)'}) 48 | .then(() => { 49 | dispatch(setTransformationJs(undefined)); 50 | dispatch(openAlert(`Reset success. No transformation will be applied to ${selectedEdge}.`)); 51 | }) 52 | }> 53 | 54 | 55 | 56 | 57 | 58 | {alertIsOpen} 59 | 60 | 61 | { 65 | request.uri = '/api/v2/' + request.uri; 66 | request.headers['api-v2-token'] = 'test-token'; 67 | return request; 68 | } 69 | ` 70 | } 71 | mode="javascript" 72 | theme="github" 73 | name="ace-editor" 74 | onLoad={()=>{}} 75 | onChange={(value) => {dispatch(setTransformationJs(value))}} 76 | fontSize={14} 77 | showPrintMargin={true} 78 | showGutter={true} 79 | highlightActiveLine={true} 80 | value={txJs} 81 | setOptions={{ 82 | enableBasicAutocompletion: true, 83 | enableLiveAutocompletion: true, 84 | enableSnippets: false, 85 | showLineNumbers: true, 86 | tabSize: 2, 87 | }}/> 88 | ; 89 | } -------------------------------------------------------------------------------- /frontend/src/features/overrides/overrideSlice.ts: -------------------------------------------------------------------------------- 1 | import {createSlice, PayloadAction} from '@reduxjs/toolkit'; 2 | interface OverrideSelections { 3 | overrideViewIsOpen: boolean, // Override field with Javascript function 4 | alertIsOpen: string|undefined, 5 | highlightedEdge: string|undefined 6 | selectedEdge: string|undefined, 7 | currentTransformationJs: string| undefined 8 | } 9 | const initialState: OverrideSelections = { 10 | overrideViewIsOpen: false, 11 | alertIsOpen: undefined, 12 | highlightedEdge: undefined, 13 | selectedEdge: undefined, 14 | currentTransformationJs: undefined 15 | }; 16 | const slice = createSlice({ 17 | name: 'overrides', 18 | initialState, 19 | reducers: { 20 | openOverrideView(state){ 21 | state.overrideViewIsOpen = true; 22 | }, 23 | closeOverrideView(state){ 24 | state.overrideViewIsOpen = false; 25 | state.alertIsOpen = undefined; 26 | state.highlightedEdge = undefined; 27 | state.selectedEdge = undefined; 28 | state.currentTransformationJs = undefined; 29 | }, 30 | clearSelections(state){ 31 | state.alertIsOpen = undefined; 32 | state.highlightedEdge = undefined; 33 | state.selectedEdge = undefined; 34 | state.currentTransformationJs = undefined; 35 | }, 36 | openAlert(state, message){ 37 | state.alertIsOpen = message.payload; 38 | }, 39 | closeAlert(state){ 40 | state.alertIsOpen = undefined; 41 | }, 42 | highlightEdge(state, edge) { 43 | state.highlightedEdge = edge.payload; 44 | }, 45 | selectEdge(state, edge) { 46 | state.selectedEdge = edge.payload; 47 | }, 48 | setTransformationJs(state, transformationJs){ 49 | state.currentTransformationJs = transformationJs.payload; 50 | } 51 | } 52 | }) 53 | 54 | export const { 55 | openOverrideView, 56 | closeOverrideView, 57 | openAlert, 58 | closeAlert, 59 | highlightEdge, 60 | selectEdge, 61 | setTransformationJs 62 | } = slice.actions; 63 | export default slice.reducer; -------------------------------------------------------------------------------- /frontend/src/features/overrides/transformationsApiSlice.ts: -------------------------------------------------------------------------------- 1 | import { createApi, fetchBaseQuery } from "@reduxjs/toolkit/query/react"; 2 | 3 | type Transformation = {injectionPoint:string, transformationJs:string} 4 | 5 | export const apiTransformationsSlice = createApi({ 6 | reducerPath: 'transformations', 7 | tagTypes: ['Transformations'], 8 | 9 | baseQuery: fetchBaseQuery({ 10 | baseUrl: '/api/1' 11 | }), 12 | endpoints(builder) { 13 | return { 14 | fetchOverride: builder.query({ 15 | query(injectionPoint){ 16 | return `/transformations/${injectionPoint}`; 17 | }, 18 | providesTags: ['Transformations'] 19 | }), 20 | updateOverride: builder.mutation({ 21 | query({injectionPoint, transformationJs}){ 22 | return { 23 | url: `/transformations/${injectionPoint}`, 24 | method: 'POST', 25 | body: transformationJs 26 | } 27 | }, 28 | invalidatesTags: ['Transformations'] 29 | }), 30 | deleteOverride: builder.mutation({ 31 | query({injectionPoint, transformationJs}){ 32 | return { 33 | url: `/transformations/${injectionPoint}`, 34 | method: 'DELETE' 35 | } 36 | }, 37 | invalidatesTags: ['Transformations'] 38 | }), 39 | } 40 | }, 41 | }); 42 | 43 | export const { 44 | useFetchOverrideQuery, 45 | useUpdateOverrideMutation, 46 | useDeleteOverrideMutation 47 | } = apiTransformationsSlice; -------------------------------------------------------------------------------- /frontend/src/features/requests/Request.ts: -------------------------------------------------------------------------------- 1 | export interface Request { 2 | request: any; 3 | left: any; 4 | right: any; 5 | } 6 | -------------------------------------------------------------------------------- /frontend/src/features/requests/RequestView.tsx: -------------------------------------------------------------------------------- 1 | import {ListItem, ListItemText, Table, TableBody,TableCell, TableContainer, TableHead, TableRow } from '@mui/material'; 2 | import { useAppSelector } from '../../app/hooks'; 3 | import { useFetchRequestQuery } from './requestsApiSlice'; 4 | 5 | export function RequestView(){ 6 | const requestId = useAppSelector((state) => state.selections.requestId) 7 | const request = requestId && useFetchRequestQuery(requestId as string).data; 8 | 9 | return (!request?No requests.: 10 | 11 | Request 12 | {
{JSON.stringify(request.request,null,4)}
} 13 |
14 | 15 | 16 | PrimaryCandidate 17 | 18 | 19 | 20 | {
{JSON.stringify(request.left,null,4)}
} 21 | {
{JSON.stringify(request.right,null,4)}
} 22 | 23 | 24 |
25 | ); 26 | } -------------------------------------------------------------------------------- /frontend/src/features/requests/requestsApiSlice.ts: -------------------------------------------------------------------------------- 1 | import { createApi, fetchBaseQuery } from "@reduxjs/toolkit/query/react"; 2 | import { Request } from "./Request"; 3 | 4 | export const apiRequestsSlice = createApi({ 5 | reducerPath: 'requests', 6 | baseQuery: fetchBaseQuery({ 7 | baseUrl: '/api/1' 8 | }), 9 | endpoints(builder) { 10 | return { 11 | fetchRequest: builder.query({ 12 | query(requestId){ 13 | return `/requests/${requestId}`; 14 | } 15 | }) 16 | } 17 | }, 18 | }); 19 | 20 | export const {useFetchRequestQuery} = apiRequestsSlice; -------------------------------------------------------------------------------- /frontend/src/features/selections/selectionsSlice.ts: -------------------------------------------------------------------------------- 1 | import {createSlice, PayloadAction} from '@reduxjs/toolkit'; 2 | type Range = [T, T] 3 | interface Selections { 4 | noiseCancellationIsOn: boolean, // Noise cancellation from AppBarView 5 | infoIsOpen: boolean, // InfoView dialog 6 | deleteRequestAlertIsOpen: boolean, // Alert that shows up inside info view when all reqests are deleted 7 | requestIsOpen: boolean, // RequestView Dialog 8 | endpointName: string|undefined, // Selected endpoint from EnpointsView 9 | fieldPrefix: string|undefined, // Selected field prefix from FieldsView 10 | requestId: string|undefined, // Selected request id from DifferencesView 11 | dateTimeRange: {start: number, end: number}, 12 | } 13 | const initialState: Selections = { 14 | noiseCancellationIsOn: false, 15 | endpointName: undefined, 16 | fieldPrefix: undefined, 17 | requestId: undefined, 18 | infoIsOpen: false, 19 | deleteRequestAlertIsOpen: false, 20 | requestIsOpen: false, 21 | dateTimeRange: {start: Date.now() - 5*60*1000, end: Date.now()} // last 5 minute 22 | }; 23 | const slice = createSlice({ 24 | name: 'selections', 25 | initialState, 26 | reducers: { 27 | toggleNoiseCancellation(state){ 28 | state.noiseCancellationIsOn = !state.noiseCancellationIsOn; 29 | }, 30 | openInfoView(state){ 31 | state.infoIsOpen = true; 32 | }, 33 | closeInfoView(state){ 34 | state.infoIsOpen = false; 35 | }, 36 | openDeleteRequestsAlert(state){ 37 | state.deleteRequestAlertIsOpen = true; 38 | }, 39 | closeDeleteRequestsAlert(state){ 40 | state.deleteRequestAlertIsOpen = false; 41 | }, 42 | openRequestView(state){ 43 | state.requestIsOpen = true; 44 | }, 45 | closeRequestView(state){ 46 | state.requestIsOpen = false; 47 | }, 48 | selectEndpoint(state, endpointName){ 49 | state.endpointName = endpointName.payload; 50 | }, 51 | selectFieldPrefix(state, fieldPrefix) { 52 | state.fieldPrefix = fieldPrefix.payload; 53 | }, 54 | selectRequest(state, requestId) { 55 | state.requestId = requestId.payload; 56 | }, 57 | setDateTimeRange(state, dateTimeRange){ 58 | state.dateTimeRange = dateTimeRange.payload; 59 | } 60 | } 61 | }) 62 | 63 | export const { 64 | toggleNoiseCancellation, 65 | openInfoView, 66 | closeInfoView, 67 | openDeleteRequestsAlert, 68 | closeDeleteRequestsAlert, 69 | openRequestView, 70 | closeRequestView, 71 | selectEndpoint, 72 | selectFieldPrefix, 73 | selectRequest, 74 | setDateTimeRange 75 | } = slice.actions; 76 | export default slice.reducer; 77 | -------------------------------------------------------------------------------- /frontend/src/main.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react' 2 | import ReactDOM from 'react-dom/client' 3 | import { Provider } from 'react-redux' 4 | import { store } from './app/store' 5 | import App from './App' 6 | 7 | ReactDOM.createRoot(document.getElementById('root') as HTMLElement).render( 8 | 9 | 10 | 11 | 12 | 13 | ) 14 | -------------------------------------------------------------------------------- /frontend/src/vite-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | -------------------------------------------------------------------------------- /frontend/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ESNext", 4 | "useDefineForClassFields": true, 5 | "lib": ["DOM", "DOM.Iterable", "ESNext"], 6 | "allowJs": false, 7 | "skipLibCheck": true, 8 | "esModuleInterop": false, 9 | "allowSyntheticDefaultImports": true, 10 | "strict": true, 11 | "forceConsistentCasingInFileNames": true, 12 | "module": "ESNext", 13 | "moduleResolution": "Node", 14 | "resolveJsonModule": true, 15 | "isolatedModules": true, 16 | "noEmit": true, 17 | "jsx": "react-jsx" 18 | }, 19 | "include": ["src"], 20 | "references": [{ "path": "./tsconfig.node.json" }] 21 | } 22 | -------------------------------------------------------------------------------- /frontend/tsconfig.node.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "composite": true, 4 | "module": "ESNext", 5 | "moduleResolution": "Node", 6 | "allowSyntheticDefaultImports": true 7 | }, 8 | "include": ["vite.config.ts"] 9 | } 10 | -------------------------------------------------------------------------------- /frontend/vite.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'vite' 2 | import react from '@vitejs/plugin-react' 3 | 4 | // https://vitejs.dev/config/ 5 | export default defineConfig({ 6 | plugins: [react()], 7 | server: { 8 | proxy : { 9 | '/api/1': 'http://localhost:8888' 10 | } 11 | } 12 | }) -------------------------------------------------------------------------------- /images/diffy-ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendiffy/diffy/6c8235b1aeb2b246b759d32a5db95a91f665a909/images/diffy-ui.png -------------------------------------------------------------------------------- /images/diffy_topology.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendiffy/diffy/6c8235b1aeb2b246b759d32a5db95a91f665a909/images/diffy_topology.png -------------------------------------------------------------------------------- /images/graphana_prometheus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendiffy/diffy/6c8235b1aeb2b246b759d32a5db95a91f665a909/images/graphana_prometheus.png -------------------------------------------------------------------------------- /images/jaeger_diffy_response_traces.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendiffy/diffy/6c8235b1aeb2b246b759d32a5db95a91f665a909/images/jaeger_diffy_response_traces.png -------------------------------------------------------------------------------- /images/loki_diffy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendiffy/diffy/6c8235b1aeb2b246b759d32a5db95a91f665a909/images/loki_diffy.png -------------------------------------------------------------------------------- /images/loki_to_tempo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendiffy/diffy/6c8235b1aeb2b246b759d32a5db95a91f665a909/images/loki_to_tempo.png -------------------------------------------------------------------------------- /images/prometheus_diffy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendiffy/diffy/6c8235b1aeb2b246b759d32a5db95a91f665a909/images/prometheus_diffy.png -------------------------------------------------------------------------------- /images/prometheus_schema_metrics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendiffy/diffy/6c8235b1aeb2b246b759d32a5db95a91f665a909/images/prometheus_schema_metrics.png -------------------------------------------------------------------------------- /images/tempo_diffy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendiffy/diffy/6c8235b1aeb2b246b759d32a5db95a91f665a909/images/tempo_diffy.png -------------------------------------------------------------------------------- /localdev.properties: -------------------------------------------------------------------------------- 1 | spring.data.mongodb.authentication-database=admin 2 | spring.data.mongodb.host=http://localhost 3 | spring.data.mongodb.port=27017 4 | spring.data.mongodb.username=root 5 | spring.data.mongodb.password=pass12345 6 | candidate=localhost:9000 7 | master.primary=localhost:9100 8 | master.secondary=localhost:9200 9 | responseMode=primary 10 | service.protocol=http 11 | allowHttpSideEffects=true 12 | serviceName=Localdev Sample Service 13 | proxy.port=8880 14 | server.port=8888 15 | rootUrl=localhost:8888 16 | otel.javaagent.debug=false 17 | otel.metrics.exporter=none 18 | otel.exporter.oltp.traces.protocol=http 19 | otel.exporter.jaeger.traces.protocol=http 20 | otel.traces.exporter=jaeger 21 | otel.exporter.jaeger.endpoint=http://localhost:14250 22 | otel.exporter.jaeger.traces.endpoint=http://localhost:14250 23 | otel.resource.attributes=service.name=diffy 24 | spring.application.name=diffy 25 | logging.level.web=INFO 26 | logging.level.io.opentelemetry=WARN 27 | logging.level.root=INFO 28 | logging.file.name=data/logs/diffy.log 29 | logging.file.max-size=10MB 30 | logging.file.max-history=1 31 | logging.file.clean-history-on-start=true 32 | logging.pattern.console=%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(%5p) %clr([diffy,%X{trace_id},%X{span_id}]) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m %n%wEx 33 | logging.pattern.file=%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(%5p) %clr([diffy,%X{trace_id},%X{span_id}]) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m %n%wEx 34 | spring.mongodb.embedded.version=4.4.13 35 | dockerComposeLocal=true 36 | management.endpoints.web.exposure.include=health,info,prometheus -------------------------------------------------------------------------------- /src/main/resources/META-INF/spring.factories: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendiffy/diffy/6c8235b1aeb2b246b759d32a5db95a91f665a909/src/main/resources/META-INF/spring.factories -------------------------------------------------------------------------------- /src/main/resources/application.yml: -------------------------------------------------------------------------------- 1 | spring.application.name: "diffy" 2 | server: 3 | port: ${http.port:8888} 4 | 5 | proxy: 6 | port: 8880 7 | 8 | candidate: "localhost:9000" 9 | 10 | master: 11 | primary: "localhost:9100" 12 | secondary: "localhost:9200" 13 | 14 | service: 15 | protocol : "http" 16 | 17 | serviceName : "Default Sample Service" 18 | 19 | spring: 20 | mongodb: 21 | embedded: 22 | version: "5.0.6" 23 | management: 24 | endpoints: 25 | web: 26 | exposure: 27 | include: health,info,prometheus 28 | 29 | logging: 30 | pattern: 31 | console: "%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(%5p) %clr([diffy,%X{trace_id},%X{span_id}]) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m %n%wEx" 32 | file: "%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(%5p) %clr([diffy,%X{trace_id},%X{span_id}]) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m %n%wEx" -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/Main.java: -------------------------------------------------------------------------------- 1 | package ai.diffy; 2 | 3 | import com.samskivert.mustache.DefaultCollector; 4 | import com.samskivert.mustache.Mustache; 5 | import org.springframework.boot.SpringApplication; 6 | import org.springframework.boot.autoconfigure.SpringBootApplication; 7 | import org.springframework.context.annotation.Bean; 8 | 9 | @SpringBootApplication 10 | public class Main { 11 | public static void main(String[] args) { 12 | SpringApplication.run(Main.class, args); 13 | } 14 | 15 | @Bean 16 | public Mustache.Compiler mustacheCompiler(Mustache.TemplateLoader templateLoader) { 17 | 18 | return Mustache.compiler() 19 | .defaultValue("Some Default Value") 20 | .withLoader(templateLoader) 21 | .withCollector(new DefaultCollector()); 22 | } 23 | } -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/NoiseController.scala: -------------------------------------------------------------------------------- 1 | package ai.diffy 2 | 3 | import ai.diffy.repository.{Noise, NoiseRepository} 4 | import org.springframework.beans.factory.annotation.Autowired 5 | import org.springframework.web.bind.annotation._ 6 | 7 | import java.util 8 | import scala.collection.mutable 9 | import scala.jdk.CollectionConverters.{ListHasAsScala, MapHasAsJava} 10 | 11 | @RestController 12 | class NoiseController(@Autowired noise: NoiseRepository) { 13 | 14 | def empty: java.util.List[String] = new java.util.ArrayList[String]() 15 | 16 | 17 | @GetMapping(path = Array("/api/1/noise")) 18 | def getAllNoise(): java.util.Map[String, java.util.List[String]] = { 19 | noise.findAll().asScala 20 | .map(endpointNoise => endpointNoise.endpoint -> endpointNoise.noisyfields) 21 | .toMap.asJava 22 | } 23 | 24 | @GetMapping(path = Array("/api/1/noise/{endpoint}")) 25 | def getNoise(@PathVariable("endpoint") endpoint: String): java.util.List[String] = { 26 | noise.findById(endpoint).map(_.noisyfields).orElse(empty) 27 | } 28 | 29 | @PostMapping(path = Array("/api/1/noise/{endpoint}/prefix/{fieldPrefix}")) 30 | def postNoise( 31 | @PathVariable("endpoint") endpoint: String, 32 | @PathVariable("fieldPrefix") fieldPrefix: String, 33 | @RequestBody mark: Mark): Boolean = { 34 | val noisyFields: java.util.List[String] = 35 | noise.findById(endpoint).map(_.noisyfields).orElse(empty) 36 | var success = false 37 | if(mark.isNoise){ 38 | success = noisyFields.add(fieldPrefix) 39 | } else { 40 | success = noisyFields.remove(fieldPrefix) 41 | } 42 | noise.save(new Noise(endpoint, noisyFields)) 43 | success 44 | } 45 | } 46 | case class Mark(isNoise: Boolean) 47 | 48 | 49 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/ObjectMapperCustomizer.scala: -------------------------------------------------------------------------------- 1 | package ai.diffy 2 | 3 | import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule 4 | import com.fasterxml.jackson.module.scala.DefaultScalaModule 5 | import org.springframework.boot.autoconfigure.jackson.Jackson2ObjectMapperBuilderCustomizer 6 | import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder 7 | import org.springframework.stereotype.Component 8 | 9 | @Component 10 | class ObjectMapperCustomizer extends Jackson2ObjectMapperBuilderCustomizer { 11 | override def customize(builder: Jackson2ObjectMapperBuilder): Unit = 12 | builder.modules(DefaultScalaModule, new JavaTimeModule) 13 | } 14 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/Renderer.scala: -------------------------------------------------------------------------------- 1 | package ai.diffy 2 | 3 | import ai.diffy.analysis.{DifferenceResult, EndpointMetadata, FieldDifference, FieldMetadata, JoinedField} 4 | import ai.diffy.lifter.JsonLifter 5 | 6 | import scala.jdk.CollectionConverters.IterableHasAsScala 7 | import scala.language.postfixOps 8 | 9 | object Renderer { 10 | def differences(diffs: Iterable[FieldDifference]) = 11 | diffs map { case fd => fd.field -> JsonLifter.decode(fd.difference) } toMap 12 | 13 | def differenceResults(drs: Iterable[DifferenceResult], includeRequestResponses: Boolean = false) = 14 | drs map { differenceResult(_, includeRequestResponses) } 15 | 16 | def differenceResult(dr: DifferenceResult, includeRequestResponses: Boolean = false) = 17 | Map( 18 | "id" -> dr.id, 19 | "trace_id" -> dr.traceId, 20 | "timestamp_msec" -> dr.timestampMsec, 21 | "endpoint" -> dr.endpoint, 22 | "differences" -> differences(dr.differences.asScala) 23 | ) ++ { 24 | if (includeRequestResponses) { 25 | Map( 26 | "request" -> JsonLifter.decode(dr.request), 27 | "left" -> JsonLifter.decode(dr.responses.primary), 28 | "right" -> JsonLifter.decode(dr.responses.candidate) 29 | ) 30 | } else { 31 | Map.empty[String, Any] 32 | } 33 | } 34 | 35 | def endpoints(endpoints: Map[String, EndpointMetadata]): Map[String, Map[String, Int]] = 36 | endpoints map { case (ep, meta) => 37 | ep -> endpoint(meta) 38 | } 39 | 40 | def endpoint(endpoint: EndpointMetadata) = Map( 41 | "total" -> endpoint.total, 42 | "differences" -> endpoint.differences 43 | ) 44 | 45 | def field(field: FieldMetadata, includeWeight: Boolean) = 46 | Map("differences" -> field.differences) ++ { 47 | if (includeWeight) { 48 | Map("weight" -> field.weight) 49 | } else { 50 | Map.empty[String, Any] 51 | } 52 | } 53 | 54 | def field(field: JoinedField, includeWeight: Boolean) = 55 | Map( 56 | "differences" -> field.raw.differences, 57 | "noise" -> field.noise.differences, 58 | "relative_difference" -> field.relativeDifference, 59 | "absolute_difference" -> field.absoluteDifference 60 | ) ++ { 61 | if (includeWeight) Map("weight" -> field.raw.weight) else Map.empty 62 | } 63 | 64 | def fields( 65 | fields: Map[String, JoinedField], 66 | includeWeight: Boolean = false 67 | ) = 68 | fields map { case (path, meta) => 69 | path -> field(meta, includeWeight) 70 | } toMap 71 | 72 | def error(message: String) = 73 | Map("error" -> message) 74 | 75 | def success(message: String) = 76 | Map("success" -> message) 77 | } -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/Settings.scala: -------------------------------------------------------------------------------- 1 | package ai.diffy 2 | 3 | import ai.diffy.functional.functions.Try 4 | import ai.diffy.util.{ResourceMatcher, ResponseMode} 5 | import org.slf4j.LoggerFactory 6 | import org.springframework.beans.factory.annotation.Value 7 | import org.springframework.stereotype.Component 8 | 9 | import java.net.URL 10 | 11 | @Component 12 | class Settings( 13 | @Value("${proxy.port}") val servicePort: Int, 14 | @Value("${candidate}") candidateAddress: String, 15 | @Value("${master.primary}") primaryAddress: String, 16 | @Value("${master.secondary}") secondaryAddress: String, 17 | @Value("${service.protocol}") val protocol: String, 18 | @Value("${serviceName}") val serviceName: String, 19 | @Value("${apiRoot:}") val apiRoot: String = "", 20 | @Value("${threshold.relative:20.0}") val relativeThreshold: Double = 20.0, 21 | @Value("${threshold.absolute:0.03}") val absoluteThreshold: Double = 0.03, 22 | @Value("${rootUrl:}") val rootUrl: String = "", 23 | @Value("${allowHttpSideEffects:false}") val allowHttpSideEffects: Boolean = false, 24 | @Value("${excludeHttpHeadersComparison:false}") val excludeHttpHeadersComparison: Boolean = false, 25 | @Value("${maxHeaderSize:8192}") val maxHeaderSize: Int = 8192, 26 | @Value("${resource.mapping:}") resourceMappings: String = "", 27 | @Value("${responseMode:primary}") mode: String = ResponseMode.primary.name(), 28 | @Value("${dockerComposeLocal:false}") val dockerComposeLocal: Boolean = false) 29 | { 30 | private[this] val log = LoggerFactory.getLogger(classOf[Settings]) 31 | val candidate = Downstream(candidateAddress) 32 | val primary = Downstream(primaryAddress) 33 | val secondary = Downstream(secondaryAddress) 34 | val resourceMatcher: Option[ResourceMatcher] = Option(resourceMappings).map( 35 | _.split(",") 36 | .map(_.split(";")) 37 | .filter { x => 38 | val wellFormed = x.length == 2 39 | if (!wellFormed) log.warn(s"Malformed resource mapping: $x. Should be ;") 40 | wellFormed 41 | } 42 | .map(x => (x(0), x(1))) 43 | .toList) 44 | .map(new ResourceMatcher(_)) 45 | 46 | val responseMode = ResponseMode.valueOf(mode); 47 | } 48 | 49 | object Downstream { 50 | def apply(address: String): Downstream = { 51 | if (Try.of(() => new URL(address)).isNormal) 52 | BaseUrl(address) 53 | else 54 | HostPort(address.split(":")(0), address.split(":")(1).toInt) 55 | } 56 | } 57 | sealed trait Downstream 58 | case class HostPort(host: String, port: Int) extends Downstream{ 59 | override def toString: String = s"${host}:${port}" 60 | } 61 | case class BaseUrl(baseUrl: String) extends Downstream { 62 | override def toString: String = baseUrl 63 | } -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/analysis/DifferenceCounter.scala: -------------------------------------------------------------------------------- 1 | package ai.diffy.analysis 2 | 3 | import ai.diffy.compare.Difference 4 | 5 | trait EndpointMetadata { 6 | // number of differences seen at this endpoint 7 | def differences: Int 8 | // total # of requests seen for this endpoint 9 | def total: Int 10 | } 11 | 12 | object FieldMetadata { 13 | val Empty = new FieldMetadata { 14 | override val differences = 0 15 | override val weight = 0 16 | } 17 | } 18 | 19 | trait FieldMetadata { 20 | // number of difference seen for this field 21 | def differences: Int 22 | // weight of this field relative to other fields, this number is calculated by counting the 23 | // number of fields that saw differences on every request that this field saw a difference in 24 | def weight: Int 25 | } 26 | 27 | trait DifferenceCounter { 28 | def count(endpoint: String, diffs: Map[String, Difference]): Unit 29 | def endpoints: Map[String, EndpointMetadata] 30 | def endpoint(endpoint: String):EndpointMetadata = endpoints(endpoint) 31 | def fields(endpoint: String): Map[String, FieldMetadata] 32 | def clear(): Unit 33 | } 34 | 35 | case class RawDifferenceCounter(counter: DifferenceCounter) 36 | case class NoiseDifferenceCounter(counter: DifferenceCounter) -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/analysis/DifferenceResult.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.analysis; 2 | 3 | import org.springframework.data.annotation.Id; 4 | import org.springframework.data.mongodb.core.index.Indexed; 5 | import org.springframework.data.mongodb.core.mapping.Document; 6 | 7 | import java.util.List; 8 | import java.util.Map; 9 | 10 | @Document 11 | public class DifferenceResult { 12 | @Id 13 | public final String id; 14 | public String traceId; 15 | public String endpoint; 16 | @Indexed 17 | public Long timestampMsec; 18 | public List differences; 19 | public String request; 20 | public Responses responses; 21 | 22 | public DifferenceResult(String id, String traceId, String endpoint, Long timestampMsec, List differences, String request, Responses responses) { 23 | this.id = id; 24 | this.traceId = traceId; 25 | this.endpoint = endpoint; 26 | this.timestampMsec = timestampMsec; 27 | this.differences = differences; 28 | this.request = request; 29 | this.responses = responses; 30 | } 31 | } 32 | 33 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/analysis/DynamicAnalyzer.scala: -------------------------------------------------------------------------------- 1 | package ai.diffy.analysis 2 | 3 | import ai.diffy.analysis.DynamicAnalyzer.decodeFieldMap 4 | import ai.diffy.lifter.{FieldMap, JsonLifter, Message} 5 | import ai.diffy.repository.DifferenceResultRepository 6 | import com.fasterxml.jackson.databind.JsonNode 7 | import com.fasterxml.jackson.databind.node.ObjectNode 8 | import scala.jdk.CollectionConverters.MapHasAsScala 9 | 10 | /** 11 | * Filters a DifferenceAnalyzer using a specified time range to output another DifferenceAnalyzer 12 | */ 13 | object DynamicAnalyzer { 14 | def decodeFieldMap(payload: String): FieldMap = { 15 | objectNodeToFieldMap(JsonLifter.decode(payload).asInstanceOf[ObjectNode]) 16 | } 17 | def objectNodeToFieldMap(objectNode: ObjectNode): FieldMap ={ 18 | val acc = new java.util.HashMap[String, Object]() 19 | objectNode.fields().forEachRemaining(entry => { 20 | acc.put(entry.getKey(), entry.getValue()) 21 | }) 22 | if(acc.containsKey("headers")) { 23 | acc.put("headers", objectNodeToFieldMap(acc.get("headers").asInstanceOf[ObjectNode])) 24 | } 25 | new FieldMap(acc.asScala.toMap) 26 | } 27 | } 28 | class DynamicAnalyzer(repository: DifferenceResultRepository) { 29 | def filter(start: Long, end: Long): Report = { 30 | val collector = new InMemoryDifferenceCollector 31 | val raw = RawDifferenceCounter(new InMemoryDifferenceCounter("raw")) 32 | val noise = NoiseDifferenceCounter(new InMemoryDifferenceCounter("noise")) 33 | val joinedDifferences = JoinedDifferences(raw, noise) 34 | val analyzer = new DifferenceAnalyzer(raw, noise, collector) 35 | 36 | val diffs = repository.findByTimestampMsecBetween(start, end) 37 | diffs.forEach(dr => { 38 | val request = Message(Some(dr.endpoint), decodeFieldMap(dr.request)) 39 | val primary = Message(Some(dr.endpoint), decodeFieldMap(dr.responses.primary)) 40 | val secondary = Message(Some(dr.endpoint), decodeFieldMap(dr.responses.secondary)) 41 | val candidate = Message(Some(dr.endpoint), decodeFieldMap(dr.responses.candidate)) 42 | analyzer.apply(request, candidate, primary, secondary, Some(dr.id)) 43 | }) 44 | Report(analyzer, joinedDifferences, collector, start, end) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/analysis/Field.scala: -------------------------------------------------------------------------------- 1 | package ai.diffy.analysis 2 | 3 | case class Field(endpoint: String, prefix: String) -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/analysis/FieldDifference.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.analysis; 2 | 3 | public class FieldDifference { 4 | public String field; 5 | public String difference; 6 | 7 | public FieldDifference(String field, String difference) { 8 | this.field = field; 9 | this.difference = difference; 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/analysis/JoinedDifferences.scala: -------------------------------------------------------------------------------- 1 | package ai.diffy.analysis 2 | 3 | import scala.math.abs 4 | 5 | object DifferencesFilterFactory { 6 | def apply(relative: Double, absolute: Double): JoinedField => Boolean = { 7 | (field: JoinedField) => 8 | field.raw.differences > field.noise.differences && 9 | field.relativeDifference > relative && 10 | field.absoluteDifference > absolute 11 | } 12 | } 13 | 14 | case class JoinedDifferences(raw: RawDifferenceCounter, noise: NoiseDifferenceCounter) { 15 | def endpoints: Map[String, JoinedEndpoint] = { 16 | raw.counter.endpoints map { case (k, _) => k -> endpoint(k) } 17 | } 18 | 19 | def endpoint(endpoint: String): JoinedEndpoint = { 20 | ( 21 | raw.counter.endpoint(endpoint), 22 | raw.counter.fields(endpoint), 23 | noise.counter.fields(endpoint) 24 | ) match { case (endpoint, rawFields, noiseFields) => 25 | JoinedEndpoint(endpoint, rawFields, noiseFields) 26 | } 27 | } 28 | } 29 | 30 | case class JoinedEndpoint( 31 | endpoint: EndpointMetadata, 32 | original: Map[String, FieldMetadata], 33 | noise: Map[String, FieldMetadata]) 34 | { 35 | def differences = endpoint.differences 36 | def total = endpoint.total 37 | def fields: Map[String, JoinedField] = original map { case (path, field) => 38 | path -> JoinedField(endpoint, field, noise.getOrElse(path, FieldMetadata.Empty)) 39 | } 40 | } 41 | 42 | case class JoinedField(endpoint: EndpointMetadata, raw: FieldMetadata, noise: FieldMetadata) { 43 | // the percent difference out of the total # of requests 44 | def absoluteDifference = abs(raw.differences - noise.differences) / endpoint.total.toDouble * 100 45 | // the square error between this field's differences and the noisy counterpart's differences 46 | def relativeDifference = abs(raw.differences - noise.differences) / (raw.differences + noise.differences).toDouble * 100 47 | } 48 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/analysis/Report.scala: -------------------------------------------------------------------------------- 1 | package ai.diffy.analysis 2 | 3 | case class Report( 4 | differenceAnalyzer: DifferenceAnalyzer, 5 | joinedDifferences: JoinedDifferences, 6 | collector: InMemoryDifferenceCollector, 7 | start: Long, 8 | end: Long) 9 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/analysis/Responses.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.analysis; 2 | 3 | public class Responses { 4 | public String primary; 5 | public String secondary; 6 | public String candidate; 7 | 8 | public Responses(String primary, String secondary, String candidate) { 9 | this.primary = primary; 10 | this.secondary = secondary; 11 | this.candidate = candidate; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/flat/FlatIndexedCollection.scala: -------------------------------------------------------------------------------- 1 | package ai.diffy.flat 2 | 3 | import scala.collection.mutable 4 | 5 | class FlatIndexedCollection { 6 | val reverseIndex = 7 | mutable.Map.empty[Seq[TerminalFlatObject], mutable.Map[FlatObject, Seq[TerminalFlatObject]]] 8 | 9 | def insert(o: FlatObject): Unit = { 10 | o.tokenizedPaths foreach { case (k, v) => 11 | reverseIndex 12 | .getOrElseUpdate(k, mutable.Map.empty[FlatObject, Seq[TerminalFlatObject]]) 13 | .update(o, v) 14 | } 15 | } 16 | 17 | def collect( 18 | path: Seq[TerminalFlatObject], 19 | predicate: TerminalFlatObject => Boolean 20 | ): Seq[FlatObject] = 21 | for { 22 | ovm <- reverseIndex.get(path).toSeq 23 | (o, vs) <- ovm.toSeq 24 | v <- vs 25 | if predicate(v) 26 | } yield { 27 | o 28 | } 29 | } 30 | 31 | sealed trait FlatCondition extends (FlatObject => Boolean) 32 | 33 | case class equals(o: FlatObject) extends FlatCondition { 34 | override def apply(other: FlatObject): Boolean = o == other 35 | } 36 | 37 | case class matches(regex: String) extends FlatCondition { 38 | override def apply(other: FlatObject): Boolean = 39 | other match { 40 | case FlatPrimitive(str: String) => str.matches(regex) 41 | case _ => false 42 | } 43 | } 44 | 45 | //case class lt[T : { def <(o:T):Boolean }](upperBound: T) extends FlatCondition { 46 | // override def apply(other: FlatObject): Boolean = 47 | // other match { 48 | // case FlatPrimitive(value: T) => value < upperBound 49 | // case _ => false 50 | // } 51 | //} 52 | // 53 | //case class gt[T<: { def >(o:T):Boolean }](lowerBound: T) extends FlatCondition { 54 | // override def apply(other: FlatObject): Boolean = 55 | // other match { 56 | // case FlatPrimitive(value: T) => value > lowerBound 57 | // case _ => false 58 | // } 59 | //} 60 | // 61 | //case class lte[T<: { def <=(o:T):Boolean }](upperBound: T) extends FlatCondition { 62 | // override def apply(other: FlatObject): Boolean = 63 | // other match { 64 | // case FlatPrimitive(value: T) => value <= upperBound 65 | // case _ => false 66 | // } 67 | //} 68 | // 69 | //case class gte[T<: { def >=(o:T):Boolean }](lowerBound: T) extends FlatCondition { 70 | // override def apply(other: FlatObject): Boolean = 71 | // other match { 72 | // case FlatPrimitive(value: T) => value >= lowerBound 73 | // case _ => false 74 | // } 75 | //} 76 | 77 | case class TerminalCondition(path: Seq[TerminalFlatObject], predicate: FlatCondition) extends FlatCondition { 78 | override def apply(o: FlatObject): Boolean = 79 | o.get(path).exists{ predicate } 80 | } 81 | 82 | case class MultipathCondition( 83 | predicate: ValueCondition, 84 | paths: Seq[TerminalFlatObject]*) 85 | extends FlatCondition { 86 | override def apply(o: FlatObject): Boolean = 87 | predicate(paths map o.get) 88 | } 89 | 90 | sealed trait ValueCondition extends (Seq[Seq[FlatObject]] => Boolean) 91 | 92 | object veq extends ValueCondition { 93 | override def apply(values: Seq[Seq[FlatObject]]): Boolean = 94 | values match { 95 | case Nil => true 96 | case Seq(head) => true 97 | case Seq(head, tail @ _*) => tail.forall{ _ == head } 98 | } 99 | } 100 | 101 | sealed trait CompositeCondition extends FlatCondition 102 | 103 | case class and(conditions: FlatCondition*) extends FlatCondition { 104 | override def apply(o:FlatObject): Boolean = conditions.forall(_(o)) 105 | } 106 | 107 | case class or(conditions: FlatCondition*) extends FlatCondition { 108 | override def apply(o:FlatObject): Boolean = conditions.exists(_(o)) 109 | } -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/algebra/Bijection.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.algebra; 2 | 3 | import java.util.Objects; 4 | 5 | public class Bijection { 6 | private final UnsafeFunction applier; 7 | private final UnsafeFunction unapplier; 8 | private Bijection(UnsafeFunction applier, UnsafeFunction unapplier) { 9 | Objects.requireNonNull(applier); 10 | Objects.requireNonNull(unapplier); 11 | 12 | this.applier = applier; 13 | this.unapplier = unapplier; 14 | } 15 | public static Bijection of(UnsafeFunction applier, UnsafeFunction unapplier) { 16 | return new Bijection<>(applier, unapplier); 17 | } 18 | public T unapply(R r) throws Throwable { 19 | return unapplier.apply(r); 20 | } 21 | 22 | public R apply(T t) throws Throwable { 23 | return applier.apply(t); 24 | } 25 | 26 | public Bijection compose(Bijection before) { 27 | Objects.requireNonNull(before); 28 | return Bijection.of(applier.compose(before.applier), unapplier.andThen(before.unapplier)); 29 | } 30 | 31 | public Bijection andThen(Bijection after) { 32 | Objects.requireNonNull(after); 33 | return Bijection.of(applier.andThen(after.applier), unapplier.compose(after.unapplier)); 34 | 35 | } 36 | 37 | public UnsafeFunction wrap(UnsafeFunction wrapped) { 38 | return applier.andThen(wrapped).andThen(unapplier); 39 | } 40 | 41 | public static Bijection identity() { 42 | return Bijection.of(t -> t, t -> t); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/algebra/UnsafeFunction.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.algebra; 2 | 3 | import java.util.Objects; 4 | import java.util.function.Function; 5 | 6 | @FunctionalInterface 7 | public interface UnsafeFunction { 8 | R apply(T t) throws Throwable; 9 | 10 | default UnsafeFunction compose(UnsafeFunction before) { 11 | Objects.requireNonNull(before); 12 | return (V v) -> apply(before.apply(v)); 13 | } 14 | 15 | default UnsafeFunction andThen(UnsafeFunction after) { 16 | Objects.requireNonNull(after); 17 | return (T t) -> after.apply(apply(t)); 18 | } 19 | 20 | default Function suppressThrowable(){ 21 | return (request) -> { 22 | try { 23 | return this.apply(request); 24 | } catch (Throwable e) { 25 | throw new RuntimeException(e); 26 | } 27 | }; 28 | } 29 | static UnsafeFunction identity() { 30 | return t -> t; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/algebra/monoids/functions/BinaryOperator.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.algebra.monoids.functions; 2 | 3 | import java.util.function.BiFunction; 4 | import java.util.function.Function; 5 | 6 | @FunctionalInterface 7 | public interface BinaryOperator< 8 | RequestIn, 9 | Request1Out, Response1In, 10 | Request2Out, Response2In, 11 | ResponseOut 12 | > extends BiFunction< 13 | Function, 14 | Function, 15 | Function 16 | > {} -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/algebra/monoids/functions/DecaOperator.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.algebra.monoids.functions; 2 | 3 | import ai.diffy.functional.functions.DecaFunction; 4 | 5 | import java.util.function.Function; 6 | 7 | /** 8 | * 9 | * @param The initial incoming request 10 | * The following endpoints may be called in any order any number of times. 11 | * NamedEndpoint, 12 | * NamedEndpoint, 13 | * NamedEndpoint, 14 | * NamedEndpoint, 15 | * NamedEndpoint, 16 | * NamedEndpoint, 17 | * NamedEndpoint, 18 | * NamedEndpoint, 19 | * NamedEndpoint, 20 | * NamedEndpoint, 21 | * The numbers 1,2,3,... only serve the purpose of enumeration and do not imply 22 | * any ordering whatsoever. 23 | * @param The final out going response 24 | */ 25 | 26 | @FunctionalInterface 27 | public interface DecaOperator< 28 | RequestIn, 29 | Request1Out, Response1In, 30 | Request2Out, Response2In, 31 | Request3Out, Response3In, 32 | Request4Out, Response4In, 33 | Request5Out, Response5In, 34 | Request6Out, Response6In, 35 | Request7Out, Response7In, 36 | Request8Out, Response8In, 37 | Request9Out, Response9In, 38 | Request10Out, Response10In, 39 | ResponseOut 40 | > extends DecaFunction< 41 | Function, 42 | Function, 43 | Function, 44 | Function, 45 | Function, 46 | Function, 47 | Function, 48 | Function, 49 | Function, 50 | Function, 51 | Function 52 | > {} 53 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/algebra/monoids/functions/HexaOperator.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.algebra.monoids.functions; 2 | 3 | import ai.diffy.functional.functions.HexaFunction; 4 | 5 | import java.util.function.Function; 6 | 7 | /** 8 | * 9 | * @param The initial incoming request 10 | * The following endpoints may be called in any order any number of times. 11 | * NamedEndpoint, 12 | * NamedEndpoint, 13 | * NamedEndpoint, 14 | * NamedEndpoint, 15 | * NamedEndpoint, 16 | * NamedEndpoint, 17 | * The numbers 1,2,3,... only serve the purpose of enumeration and do not imply 18 | * any ordering whatsoever. 19 | * @param The final out going response 20 | */ 21 | 22 | @FunctionalInterface 23 | public interface HexaOperator< 24 | RequestIn, 25 | Request1Out, Response1In, 26 | Request2Out, Response2In, 27 | Request3Out, Response3In, 28 | Request4Out, Response4In, 29 | Request5Out, Response5In, 30 | Request6Out, Response6In, 31 | ResponseOut 32 | > extends HexaFunction< 33 | Function, 34 | Function, 35 | Function, 36 | Function, 37 | Function, 38 | Function, 39 | Function 40 | > {} 41 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/algebra/monoids/functions/NonaOperator.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.algebra.monoids.functions; 2 | 3 | import ai.diffy.functional.functions.NonaFunction; 4 | 5 | import java.util.function.Function; 6 | 7 | /** 8 | * 9 | * @param The initial incoming request 10 | * The following endpoints may be called in any order any number of times. 11 | * NamedEndpoint, 12 | * NamedEndpoint, 13 | * NamedEndpoint, 14 | * NamedEndpoint, 15 | * NamedEndpoint, 16 | * NamedEndpoint, 17 | * NamedEndpoint, 18 | * NamedEndpoint, 19 | * NamedEndpoint, 20 | * The numbers 1,2,3,... only serve the purpose of enumeration and do not imply 21 | * any ordering whatsoever. 22 | * @param The final out going response 23 | */ 24 | 25 | @FunctionalInterface 26 | public interface NonaOperator< 27 | RequestIn, 28 | Request1Out, Response1In, 29 | Request2Out, Response2In, 30 | Request3Out, Response3In, 31 | Request4Out, Response4In, 32 | Request5Out, Response5In, 33 | Request6Out, Response6In, 34 | Request7Out, Response7In, 35 | Request8Out, Response8In, 36 | Request9Out, Response9In, 37 | ResponseOut 38 | > extends NonaFunction< 39 | Function, 40 | Function, 41 | Function, 42 | Function, 43 | Function, 44 | Function, 45 | Function, 46 | Function, 47 | Function, 48 | Function 49 | > {} 50 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/algebra/monoids/functions/NullOperator.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.algebra.monoids.functions; 2 | 3 | import java.util.function.Function; 4 | import java.util.function.Supplier; 5 | 6 | @FunctionalInterface 7 | public interface NullOperator< 8 | RequestIn, 9 | ResponseOut 10 | > extends Supplier< 11 | Function // Inbound Ingress call 12 | > {} 13 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/algebra/monoids/functions/OctaOperator.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.algebra.monoids.functions; 2 | 3 | import ai.diffy.functional.functions.OctaFunction; 4 | 5 | import java.util.function.Function; 6 | 7 | /** 8 | * 9 | * @param The initial incoming request 10 | * The following endpoints may be called in any order any number of times. 11 | * NamedEndpoint, 12 | * NamedEndpoint, 13 | * NamedEndpoint, 14 | * NamedEndpoint, 15 | * NamedEndpoint, 16 | * NamedEndpoint, 17 | * NamedEndpoint, 18 | * NamedEndpoint, 19 | * The numbers 1,2,3,... only serve the purpose of enumeration and do not imply 20 | * any ordering whatsoever. 21 | * @param The final out going response 22 | */ 23 | 24 | @FunctionalInterface 25 | public interface OctaOperator< 26 | RequestIn, 27 | Request1Out, Response1In, 28 | Request2Out, Response2In, 29 | Request3Out, Response3In, 30 | Request4Out, Response4In, 31 | Request5Out, Response5In, 32 | Request6Out, Response6In, 33 | Request7Out, Response7In, 34 | Request8Out, Response8In, 35 | ResponseOut 36 | > extends OctaFunction< 37 | Function, 38 | Function, 39 | Function, 40 | Function, 41 | Function, 42 | Function, 43 | Function, 44 | Function, 45 | Function 46 | > {} 47 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/algebra/monoids/functions/PentaOperator.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.algebra.monoids.functions; 2 | 3 | import ai.diffy.functional.functions.PentaFunction; 4 | 5 | import java.util.function.Function; 6 | 7 | /** 8 | * 9 | * @param The initial incoming request 10 | * The following endpoints may be called in any order any number of times. 11 | * NamedEndpoint, 12 | * NamedEndpoint, 13 | * NamedEndpoint, 14 | * NamedEndpoint, 15 | * NamedEndpoint, 16 | * The numbers 1,2,3,... only serve the purpose of enumeration and do not imply 17 | * any ordering whatsoever. 18 | * @param The final out going response 19 | */ 20 | 21 | @FunctionalInterface 22 | public interface PentaOperator< 23 | RequestIn, 24 | Request1Out, Response1In, 25 | Request2Out, Response2In, 26 | Request3Out, Response3In, 27 | Request4Out, Response4In, 28 | Request5Out, Response5In, 29 | ResponseOut 30 | > extends PentaFunction< 31 | Function, 32 | Function, 33 | Function, 34 | Function, 35 | Function, 36 | Function 37 | > {} 38 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/algebra/monoids/functions/QuadOperator.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.algebra.monoids.functions; 2 | 3 | import ai.diffy.functional.functions.QuadFunction; 4 | 5 | import java.util.function.Function; 6 | 7 | /** 8 | * 9 | * @param The initial incoming request 10 | * The following services may be called in any order any number of times. 11 | * @param @param 12 | * @param @param 13 | * @param @param 14 | * @param @param 15 | * The numbers 1,2,3,4 only serve the purpose of enumeration and do not imply 16 | * any ordering whatsoever. 17 | * @param The final out going response 18 | */ 19 | 20 | @FunctionalInterface 21 | public interface QuadOperator< 22 | RequestIn, 23 | Request1Out, Response1In, 24 | Request2Out, Response2In, 25 | Request3Out, Response3In, 26 | Request4Out, Response4In, 27 | ResponseOut 28 | > extends QuadFunction< 29 | Function, 30 | Function, 31 | Function, 32 | Function, 33 | Function 34 | > {} 35 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/algebra/monoids/functions/SeptaOperator.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.algebra.monoids.functions; 2 | 3 | import ai.diffy.functional.functions.SeptaFunction; 4 | 5 | import java.util.function.Function; 6 | 7 | /** 8 | * 9 | * @param The initial incoming request 10 | * The following endpoints may be called in any order any number of times. 11 | * NamedEndpoint, 12 | * NamedEndpoint, 13 | * NamedEndpoint, 14 | * NamedEndpoint, 15 | * NamedEndpoint, 16 | * NamedEndpoint, 17 | * NamedEndpoint, 18 | * The numbers 1,2,3,... only serve the purpose of enumeration and do not imply 19 | * any ordering whatsoever. 20 | * @param The final out going response 21 | */ 22 | 23 | @FunctionalInterface 24 | public interface SeptaOperator< 25 | RequestIn, 26 | Request1Out, Response1In, 27 | Request2Out, Response2In, 28 | Request3Out, Response3In, 29 | Request4Out, Response4In, 30 | Request5Out, Response5In, 31 | Request6Out, Response6In, 32 | Request7Out, Response7In, 33 | ResponseOut 34 | > extends SeptaFunction< 35 | Function, 36 | Function, 37 | Function, 38 | Function, 39 | Function, 40 | Function, 41 | Function, 42 | Function 43 | > {} 44 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/algebra/monoids/functions/SymmetricUnaryOperator.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.algebra.monoids.functions; 2 | 3 | import java.util.Objects; 4 | 5 | @FunctionalInterface 6 | public interface SymmetricUnaryOperator extends UnaryOperator { 7 | 8 | default SymmetricUnaryOperator compose(SymmetricUnaryOperator before) { 9 | Objects.requireNonNull(before); 10 | return (applier) -> apply(before.apply(applier)); 11 | } 12 | default SymmetricUnaryOperator andThen(SymmetricUnaryOperator after) { 13 | Objects.requireNonNull(after); 14 | return (applier) -> after.apply(apply(applier)); 15 | } 16 | static SymmetricUnaryOperator identity() { 17 | return t -> t; 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/algebra/monoids/functions/TernaryOperator.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.algebra.monoids.functions; 2 | 3 | import ai.diffy.functional.functions.TriFunction; 4 | 5 | import java.util.function.Function; 6 | 7 | @FunctionalInterface 8 | public interface TernaryOperator< 9 | RequestIn, 10 | Request1Out, Response1In, 11 | Request2Out, Response2In, 12 | Request3Out, Response3In, 13 | ResponseOut 14 | > extends TriFunction< 15 | Function, 16 | Function, 17 | Function, 18 | Function 19 | > {} -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/algebra/monoids/functions/UnaryOperator.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.algebra.monoids.functions; 2 | 3 | import java.util.function.Function; 4 | 5 | @FunctionalInterface 6 | public interface UnaryOperator< 7 | RequestIn, 8 | RequestOut, ResponseIn, 9 | ResponseOut 10 | > extends Function< 11 | Function, // Downstream Egress call 12 | Function // Inbound Ingress call 13 | > {} 14 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/algebra/monoids/suppliers/BiSupplierOperator.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.algebra.monoids.suppliers; 2 | 3 | import java.util.function.BiFunction; 4 | import java.util.function.Supplier; 5 | 6 | @FunctionalInterface 7 | public interface BiSupplierOperator extends BiFunction, Supplier, Supplier> { } 8 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/algebra/monoids/suppliers/NullSupplierOperator.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.algebra.monoids.suppliers; 2 | 3 | import java.util.function.Supplier; 4 | 5 | @FunctionalInterface 6 | public interface NullSupplierOperator extends Supplier> { } 7 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/algebra/monoids/suppliers/QuadSupplierOperator.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.algebra.monoids.suppliers; 2 | 3 | import ai.diffy.functional.functions.QuadFunction; 4 | 5 | import java.util.function.Supplier; 6 | 7 | @FunctionalInterface 8 | public interface QuadSupplierOperator 9 | extends QuadFunction< 10 | Supplier, 11 | Supplier, 12 | Supplier, 13 | Supplier, 14 | Supplier> { } 15 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/algebra/monoids/suppliers/TriSupplierOperator.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.algebra.monoids.suppliers; 2 | 3 | import ai.diffy.functional.functions.TriFunction; 4 | 5 | import java.util.function.Supplier; 6 | 7 | @FunctionalInterface 8 | public interface TriSupplierOperator extends TriFunction, Supplier, Supplier, Supplier> { } 9 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/algebra/monoids/suppliers/UnarySupplierOperator.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.algebra.monoids.suppliers; 2 | 3 | import java.util.function.Function; 4 | import java.util.function.Supplier; 5 | 6 | @FunctionalInterface 7 | public interface UnarySupplierOperator extends Function, Supplier> { } 8 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/algebra/unions/SFFBinaryOperator.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.algebra.unions; 2 | 3 | import java.util.function.BiFunction; 4 | import java.util.function.Function; 5 | import java.util.function.Supplier; 6 | 7 | @FunctionalInterface 8 | public interface SFFBinaryOperator extends BiFunction, Function, Function> {} 9 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/algebra/unions/SFSBinaryOperator.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.algebra.unions; 2 | 3 | import java.util.function.BiFunction; 4 | import java.util.function.Function; 5 | import java.util.function.Supplier; 6 | 7 | @FunctionalInterface 8 | public interface SFSBinaryOperator extends BiFunction, Function, Supplier> { } 9 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/endpoints/BiDependentEndpoint.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.endpoints; 2 | 3 | import ai.diffy.functional.algebra.monoids.functions.BinaryOperator; 4 | 5 | import java.util.List; 6 | 7 | public class BiDependentEndpoint< 8 | RequestIn, 9 | Request1, Response1, 10 | Request2, Response2, 11 | ResponseOut> extends Endpoint { 12 | 13 | private final Endpoint dependency1; 14 | private final Endpoint dependency2; 15 | 16 | private final BinaryOperator filter; 20 | protected BiDependentEndpoint( 21 | String name, 22 | Endpoint dependency1, 23 | Endpoint dependency2, 24 | BinaryOperator filter) { 28 | super(name, filter.apply(dependency1, dependency2)); 29 | this.dependency1 = dependency1; 30 | this.dependency2 = dependency2; 31 | this.filter = filter; 32 | } 33 | 34 | @Override 35 | public List getDownstream() { 36 | return List.of(dependency1, dependency2); 37 | } 38 | 39 | @Override 40 | public Endpoint deepClone() { 41 | return new BiDependentEndpoint<>( 42 | this.name, 43 | dependency1.deepClone(), 44 | dependency2.deepClone(), 45 | filter).setMiddleware(this.getMiddleware()); 46 | } 47 | 48 | @Override 49 | public Endpoint withDownstream(List downstream) { 50 | assert downstream.size() == 2; 51 | assert this.dependency1.getClass().isAssignableFrom(downstream.get(0).getClass()); 52 | assert this.dependency2.getClass().isAssignableFrom(downstream.get(1).getClass()); 53 | return new BiDependentEndpoint<>( 54 | this.name, 55 | downstream.get(0), 56 | downstream.get(1), 57 | filter).setMiddleware(this.getMiddleware()); 58 | } 59 | } -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/endpoints/DependentEndpoint.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.endpoints; 2 | 3 | import ai.diffy.functional.algebra.monoids.functions.UnaryOperator; 4 | 5 | import java.util.List; 6 | 7 | public class DependentEndpoint< 8 | RequestIn, 9 | RequestOut, ResponseIn, 10 | ResponseOut> extends Endpoint { 11 | private final Endpoint dependency; 12 | private final UnaryOperator unaryOperator; 15 | protected DependentEndpoint( 16 | String name, 17 | Endpoint dependency, 18 | UnaryOperator unaryOperator) { 21 | super(name, unaryOperator.apply(dependency)); 22 | this.dependency = dependency; 23 | this.unaryOperator = unaryOperator; 24 | } 25 | 26 | @Override 27 | public List getDownstream() { 28 | return List.of(dependency); 29 | } 30 | 31 | @Override 32 | public Endpoint deepClone() { 33 | return new DependentEndpoint<>( 34 | this.name, 35 | dependency.deepClone(), 36 | unaryOperator).setMiddleware(this.getMiddleware()); 37 | } 38 | 39 | @Override 40 | public Endpoint withDownstream(List downstream) { 41 | assert downstream.size() == 1; 42 | assert this.dependency.getClass().isAssignableFrom(downstream.get(0).getClass()); 43 | return new DependentEndpoint<>(this.name, downstream.get(0), this.unaryOperator) 44 | .setMiddleware(this.getMiddleware()); 45 | } 46 | } -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/endpoints/HexaDependentEndpoint.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.endpoints; 2 | 3 | import ai.diffy.functional.algebra.monoids.functions.HexaOperator; 4 | 5 | import java.util.List; 6 | 7 | public class HexaDependentEndpoint< 8 | RequestIn, 9 | Request1, Response1, 10 | Request2, Response2, 11 | Request3, Response3, 12 | Request4, Response4, 13 | Request5, Response5, 14 | Request6, Response6, 15 | ResponseOut> extends Endpoint { 16 | private final Endpoint dependency1; 17 | private final Endpoint dependency2; 18 | private final Endpoint dependency3; 19 | private final Endpoint dependency4; 20 | private final Endpoint dependency5; 21 | private final Endpoint dependency6; 22 | private final HexaOperator filter; 30 | protected HexaDependentEndpoint( 31 | String name, 32 | Endpoint dependency1, 33 | Endpoint dependency2, 34 | Endpoint dependency3, 35 | Endpoint dependency4, 36 | Endpoint dependency5, 37 | Endpoint dependency6, 38 | HexaOperator filter) { 46 | super(name, filter.apply( 47 | dependency1, 48 | dependency2, 49 | dependency3, 50 | dependency4, 51 | dependency5, 52 | dependency6 53 | )); 54 | this.dependency1 = dependency1; 55 | this.dependency2 = dependency2; 56 | this.dependency3 = dependency3; 57 | this.dependency4 = dependency4; 58 | this.dependency5 = dependency5; 59 | this.dependency6 = dependency6; 60 | this.filter = filter; 61 | } 62 | 63 | @Override 64 | public List getDownstream() { 65 | return List.of(dependency1, dependency2, dependency3, dependency4, dependency5, dependency6); 66 | } 67 | 68 | @Override 69 | public Endpoint deepClone() { 70 | return new HexaDependentEndpoint<>( 71 | this.name, 72 | dependency1.deepClone(), 73 | dependency2.deepClone(), 74 | dependency3.deepClone(), 75 | dependency4.deepClone(), 76 | dependency5.deepClone(), 77 | dependency6.deepClone(), 78 | filter 79 | ).setMiddleware(this.getMiddleware()); 80 | } 81 | 82 | @Override 83 | public Endpoint withDownstream(List downstream) { 84 | assert downstream.size() == 6; 85 | assert this.dependency1.getClass().isAssignableFrom(downstream.get(0).getClass()); 86 | assert this.dependency2.getClass().isAssignableFrom(downstream.get(1).getClass()); 87 | assert this.dependency3.getClass().isAssignableFrom(downstream.get(2).getClass()); 88 | assert this.dependency4.getClass().isAssignableFrom(downstream.get(3).getClass()); 89 | assert this.dependency5.getClass().isAssignableFrom(downstream.get(4).getClass()); 90 | assert this.dependency6.getClass().isAssignableFrom(downstream.get(5).getClass()); 91 | return new HexaDependentEndpoint<>( 92 | this.name, 93 | downstream.get(0), 94 | downstream.get(1), 95 | downstream.get(2), 96 | downstream.get(3), 97 | downstream.get(4), 98 | downstream.get(5), 99 | filter).setMiddleware(this.getMiddleware()); 100 | } 101 | } -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/endpoints/IndependentEndpoint.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.endpoints; 2 | 3 | import ai.diffy.functional.algebra.monoids.functions.NullOperator; 4 | 5 | import java.util.List; 6 | 7 | public class IndependentEndpoint extends Endpoint{ 8 | private final NullOperator filter; 9 | protected IndependentEndpoint(String name, NullOperator filter) { 10 | super(name, filter.get()); 11 | this.filter = filter; 12 | } 13 | 14 | @Override 15 | public List getDownstream() { 16 | return List.of(); 17 | } 18 | 19 | @Override 20 | public Endpoint deepClone() { 21 | return new IndependentEndpoint<>(this.name, this.filter).setMiddleware(this.getMiddleware()); 22 | } 23 | 24 | @Override 25 | public Endpoint withDownstream(List downstream) { 26 | assert downstream.isEmpty(); 27 | return deepClone().setMiddleware(this.getMiddleware()); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/endpoints/PentaDependentEndpoint.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.endpoints; 2 | 3 | import ai.diffy.functional.algebra.monoids.functions.PentaOperator; 4 | 5 | import java.util.List; 6 | 7 | public class PentaDependentEndpoint< 8 | RequestIn, 9 | Request1, Response1, 10 | Request2, Response2, 11 | Request3, Response3, 12 | Request4, Response4, 13 | Request5, Response5, 14 | ResponseOut> extends Endpoint { 15 | private final Endpoint dependency1; 16 | private final Endpoint dependency2; 17 | private final Endpoint dependency3; 18 | private final Endpoint dependency4; 19 | private final Endpoint dependency5; 20 | private final PentaOperator filter; 27 | protected PentaDependentEndpoint( 28 | String name, 29 | Endpoint dependency1, 30 | Endpoint dependency2, 31 | Endpoint dependency3, 32 | Endpoint dependency4, 33 | Endpoint dependency5, 34 | PentaOperator filter) { 41 | super(name, filter.apply(dependency1, dependency2, dependency3, dependency4, dependency5)); 42 | this.dependency1 = dependency1; 43 | this.dependency2 = dependency2; 44 | this.dependency3 = dependency3; 45 | this.dependency4 = dependency4; 46 | this.dependency5 = dependency5; 47 | this.filter = filter; 48 | } 49 | 50 | @Override 51 | public List getDownstream() { 52 | return List.of(dependency1, dependency2, dependency3, dependency4, dependency5); 53 | } 54 | 55 | @Override 56 | public Endpoint deepClone() { 57 | return new PentaDependentEndpoint<>( 58 | this.name, 59 | dependency1.deepClone(), 60 | dependency2.deepClone(), 61 | dependency3.deepClone(), 62 | dependency4.deepClone(), 63 | dependency5.deepClone(), 64 | filter 65 | ).setMiddleware(this.getMiddleware()); 66 | } 67 | 68 | @Override 69 | public Endpoint withDownstream(List downstream) { 70 | assert downstream.size() == 5; 71 | assert this.dependency1.getClass().isAssignableFrom(downstream.get(0).getClass()); 72 | assert this.dependency2.getClass().isAssignableFrom(downstream.get(1).getClass()); 73 | assert this.dependency3.getClass().isAssignableFrom(downstream.get(2).getClass()); 74 | assert this.dependency4.getClass().isAssignableFrom(downstream.get(3).getClass()); 75 | assert this.dependency5.getClass().isAssignableFrom(downstream.get(4).getClass()); 76 | return new PentaDependentEndpoint<>( 77 | this.name, 78 | downstream.get(0), 79 | downstream.get(1), 80 | downstream.get(2), 81 | downstream.get(3), 82 | downstream.get(4), 83 | filter).setMiddleware(this.getMiddleware()); 84 | } 85 | } -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/endpoints/QuadDependentEndpoint.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.endpoints; 2 | 3 | import ai.diffy.functional.algebra.monoids.functions.QuadOperator; 4 | 5 | import java.util.List; 6 | 7 | public class QuadDependentEndpoint< 8 | RequestIn, 9 | Request1, Response1, 10 | Request2, Response2, 11 | Request3, Response3, 12 | Request4, Response4, 13 | ResponseOut> extends Endpoint { 14 | private final Endpoint dependency1; 15 | private final Endpoint dependency2; 16 | private final Endpoint dependency3; 17 | private final Endpoint dependency4; 18 | private final QuadOperator filter; 24 | protected QuadDependentEndpoint( 25 | String name, 26 | Endpoint dependency1, 27 | Endpoint dependency2, 28 | Endpoint dependency3, 29 | Endpoint dependency4, 30 | QuadOperator filter) { 36 | super(name, filter.apply(dependency1, dependency2, dependency3, dependency4)); 37 | this.dependency1 = dependency1; 38 | this.dependency2 = dependency2; 39 | this.dependency3 = dependency3; 40 | this.dependency4 = dependency4; 41 | this.filter = filter; 42 | } 43 | 44 | @Override 45 | public List getDownstream() { 46 | return List.of(dependency1, dependency2, dependency3, dependency4); 47 | } 48 | 49 | @Override 50 | public Endpoint deepClone() { 51 | return new QuadDependentEndpoint<>( 52 | this.name, 53 | dependency1.deepClone(), 54 | dependency2.deepClone(), 55 | dependency3.deepClone(), 56 | dependency4.deepClone(), 57 | filter).setMiddleware(this.getMiddleware()); 58 | } 59 | 60 | @Override 61 | public Endpoint withDownstream(List downstream) { 62 | assert downstream.size() == 4; 63 | assert this.dependency1.getClass().isAssignableFrom(downstream.get(0).getClass()); 64 | assert this.dependency2.getClass().isAssignableFrom(downstream.get(1).getClass()); 65 | assert this.dependency3.getClass().isAssignableFrom(downstream.get(2).getClass()); 66 | assert this.dependency4.getClass().isAssignableFrom(downstream.get(3).getClass()); 67 | return new QuadDependentEndpoint<>( 68 | this.name, 69 | downstream.get(0), 70 | downstream.get(1), 71 | downstream.get(2), 72 | downstream.get(3), 73 | filter).setMiddleware(this.getMiddleware()); 74 | } 75 | } -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/endpoints/SymmetricDependentEndpoint.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.endpoints; 2 | 3 | import ai.diffy.functional.algebra.monoids.functions.UnaryOperator; 4 | 5 | public class SymmetricDependentEndpoint 6 | extends DependentEndpoint { 7 | public SymmetricDependentEndpoint( 8 | String name, 9 | Endpoint dependency, 10 | UnaryOperator unaryOperator) { 11 | super(name, dependency, unaryOperator); 12 | } 13 | } -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/endpoints/TriDependentEndpoint.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.endpoints; 2 | 3 | import ai.diffy.functional.algebra.monoids.functions.TernaryOperator; 4 | 5 | import java.util.List; 6 | 7 | public class TriDependentEndpoint< 8 | RequestIn, 9 | Request1, Response1, 10 | Request2, Response2, 11 | Request3, Response3, 12 | ResponseOut> extends Endpoint { 13 | 14 | private final Endpoint dependency1; 15 | private final Endpoint dependency2; 16 | private final Endpoint dependency3; 17 | 18 | private final TernaryOperator filter; 23 | protected TriDependentEndpoint( 24 | String name, 25 | Endpoint dependency1, 26 | Endpoint dependency2, 27 | Endpoint dependency3, 28 | TernaryOperator filter) { 33 | super(name, filter.apply(dependency1, dependency2, dependency3)); 34 | this.dependency1 = dependency1; 35 | this.dependency2 = dependency2; 36 | this.dependency3 = dependency3; 37 | this.filter = filter; 38 | } 39 | 40 | @Override 41 | public List getDownstream() { 42 | return List.of(dependency1, dependency2, dependency3); 43 | } 44 | 45 | @Override 46 | public Endpoint deepClone() { 47 | return new TriDependentEndpoint<>( 48 | this.name, 49 | dependency1.deepClone(), 50 | dependency2.deepClone(), 51 | dependency3.deepClone(), 52 | filter).setMiddleware(this.getMiddleware()); 53 | } 54 | @Override 55 | public Endpoint withDownstream(List downstream) { 56 | assert downstream.size() == 3; 57 | assert this.dependency1.getClass().isAssignableFrom(downstream.get(0).getClass()); 58 | assert this.dependency2.getClass().isAssignableFrom(downstream.get(1).getClass()); 59 | assert this.dependency3.getClass().isAssignableFrom(downstream.get(2).getClass()); 60 | return new TriDependentEndpoint<>( 61 | this.name, 62 | downstream.get(0), 63 | downstream.get(1), 64 | downstream.get(2), 65 | filter).setMiddleware(this.getMiddleware()); 66 | } 67 | } -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/functions/DecaFunction.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.functions; 2 | 3 | import java.util.Objects; 4 | import java.util.function.Function; 5 | 6 | @FunctionalInterface 7 | public interface DecaFunction { 8 | 9 | Z apply(A a, B b, C c, D d, E e, F f, G g, H h, I i, J j); 10 | 11 | default DecaFunction andThen(Function after) { 12 | Objects.requireNonNull(after); 13 | return (A a, B b, C c, D d, E e, F f, G g, H h, I i, J j) -> after.apply(apply(a,b,c,d,e,f,g,h,i,j)); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/functions/HexaFunction.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.functions; 2 | 3 | import java.util.Objects; 4 | import java.util.function.Function; 5 | 6 | @FunctionalInterface 7 | public interface HexaFunction { 8 | 9 | Z apply(A a, B b, C c, D d, E e, F f); 10 | 11 | default HexaFunction andThen(Function after) { 12 | Objects.requireNonNull(after); 13 | return (A a, B b, C c, D d, E e, F f) -> after.apply(apply(a,b,c,d,e,f)); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/functions/NonaFunction.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.functions; 2 | 3 | import java.util.Objects; 4 | import java.util.function.Function; 5 | 6 | @FunctionalInterface 7 | public interface NonaFunction { 8 | 9 | Z apply(A a, B b, C c, D d, E e, F f, G g, H h, I i); 10 | 11 | default NonaFunction andThen(Function after) { 12 | Objects.requireNonNull(after); 13 | return (A a, B b, C c, D d, E e, F f, G g, H h, I i) -> after.apply(apply(a,b,c,d,e,f,g,h,i)); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/functions/OctaFunction.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.functions; 2 | 3 | import java.util.Objects; 4 | import java.util.function.Function; 5 | 6 | @FunctionalInterface 7 | public interface OctaFunction { 8 | 9 | Z apply(A a, B b, C c, D d, E e, F f, G g, H h); 10 | 11 | default OctaFunction andThen(Function after) { 12 | Objects.requireNonNull(after); 13 | return (A a, B b, C c, D d, E e, F f, G g, H h) -> after.apply(apply(a,b,c,d,e,f,g,h)); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/functions/PentaFunction.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.functions; 2 | 3 | import java.util.Objects; 4 | import java.util.function.Function; 5 | 6 | @FunctionalInterface 7 | public interface PentaFunction { 8 | 9 | Z apply(A a, B b, C c, D d, E e); 10 | 11 | default PentaFunction andThen(Function after) { 12 | Objects.requireNonNull(after); 13 | return (A a, B b, C c, D d, E e) -> after.apply(apply(a,b,c,d,e)); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/functions/Proceeder.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.functions; 2 | 3 | @FunctionalInterface 4 | public interface Proceeder { 5 | T get() throws Throwable; 6 | } -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/functions/QuadFunction.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.functions; 2 | 3 | import java.util.Objects; 4 | import java.util.function.Function; 5 | 6 | @FunctionalInterface 7 | public interface QuadFunction { 8 | 9 | Z apply(A a, B b, C c, D d); 10 | 11 | default QuadFunction andThen(Function after) { 12 | Objects.requireNonNull(after); 13 | return (A a, B b, C c, D d) -> after.apply(apply(a,b,c,d)); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/functions/SeptaFunction.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.functions; 2 | 3 | import java.util.Objects; 4 | import java.util.function.Function; 5 | 6 | @FunctionalInterface 7 | public interface SeptaFunction { 8 | 9 | Z apply(A a, B b, C c, D d, E e, F f, G g); 10 | 11 | default SeptaFunction andThen(Function after) { 12 | Objects.requireNonNull(after); 13 | return (A a, B b, C c, D d, E e, F f, G g) -> after.apply(apply(a,b,c,d,e,f,g)); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/functions/TriFunction.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.functions; 2 | 3 | import java.util.Objects; 4 | import java.util.function.Function; 5 | 6 | @FunctionalInterface 7 | public interface TriFunction { 8 | 9 | Z apply(A a, B b, C c); 10 | 11 | default TriFunction andThen(Function after) { 12 | Objects.requireNonNull(after); 13 | return (A a, B b, C c) -> after.apply(apply(a,b,c)); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/functions/Try.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.functions; 2 | 3 | import java.util.concurrent.CompletableFuture; 4 | import java.util.function.Function; 5 | 6 | public class Try { 7 | private T normal; 8 | private Throwable throwable; 9 | 10 | public static Try of(Proceeder proceeder){ 11 | return new Try<>(proceeder); 12 | } 13 | public Boolean isNormal(){ 14 | return this.throwable == null; 15 | } 16 | 17 | public T get() throws Throwable { 18 | if(isNormal()){ 19 | return this.normal; 20 | } 21 | throw this.throwable; 22 | } 23 | 24 | public Try(Proceeder proceeder) { 25 | try { 26 | this.normal = proceeder.get(); 27 | } catch (Throwable throwable){ 28 | this.throwable = throwable; 29 | } 30 | } 31 | 32 | public T getNormal() { 33 | return normal; 34 | } 35 | 36 | public Throwable getThrowable() { 37 | return throwable; 38 | } 39 | 40 | public CompletableFuture toFuture(){ 41 | if (isNormal()) { 42 | return CompletableFuture.completedFuture(getNormal()); 43 | } 44 | return CompletableFuture.failedFuture(getThrowable()); 45 | } 46 | 47 | public Try map(Function mapper) { 48 | return Try.of(() -> mapper.apply(this.get())); 49 | } 50 | 51 | @Override 52 | public String toString() { 53 | return String.format("try = {normal = %s, throwable = %s}", normal, throwable); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/topology/Async.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.topology; 2 | 3 | import ai.diffy.functional.endpoints.DependentEndpoint; 4 | import ai.diffy.functional.endpoints.Endpoint; 5 | import ai.diffy.functional.functions.Try; 6 | import ai.diffy.util.Future; 7 | 8 | import java.util.concurrent.CompletableFuture; 9 | import java.util.concurrent.ForkJoinPool; 10 | import java.util.function.Function; 11 | 12 | /** 13 | * AsyncEndpointWrapper wraps a synchronous NamedEndpoint with a ForkJoinPool to produce an async interface 14 | * @param 15 | * @param 16 | */ 17 | public class Async extends DependentEndpoint> { 18 | public static Function> operator(ForkJoinPool pool, Function applyDependency) { 19 | return (Request request) -> { 20 | CompletableFuture result = new CompletableFuture<>(); 21 | pool.execute(()-> { 22 | Try x = Try.of(() -> result.complete(applyDependency.apply(request))); 23 | if(!x.isNormal()){ 24 | result.completeExceptionally(x.getThrowable()); 25 | } 26 | assert result.isDone(); 27 | }); 28 | return result; 29 | }; 30 | } 31 | public static Async common(Endpoint dependency) { 32 | return new Async<>(ForkJoinPool.commonPool(), dependency); 33 | } 34 | public static Endpoint> contain(Endpoint> dependency) { 35 | return contain(ForkJoinPool.commonPool(), dependency); 36 | } 37 | public static Endpoint> contain(ForkJoinPool pool, Endpoint> dependency) { 38 | return Endpoint.from( 39 | dependency.getName()+".contained", 40 | () -> (Request request) -> { 41 | final CompletableFuture result = new CompletableFuture<>(); 42 | pool.execute(()-> { 43 | Future.assign( 44 | Try.of(() -> dependency.apply(request)) 45 | .toFuture() 46 | .thenCompose(Function.identity()), 47 | result 48 | ); 49 | }); 50 | return result; 51 | }); 52 | } 53 | public Async(ForkJoinPool pool, Endpoint dependency) { 54 | super(dependency.getName()+".async", dependency, applyDependency -> operator(pool, applyDependency)); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/topology/AsyncCommonPoolUnaryOperator.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.topology; 2 | 3 | import java.util.concurrent.CompletableFuture; 4 | import java.util.concurrent.ForkJoinPool; 5 | import java.util.function.Function; 6 | 7 | public interface AsyncCommonPoolUnaryOperator extends AsyncUnaryOperator { 8 | ForkJoinPool pool = ForkJoinPool.commonPool(); 9 | @Override 10 | default Function> apply(Function dependency) { 11 | return (Request request) -> { 12 | CompletableFuture result = new CompletableFuture<>(); 13 | pool.execute(() -> result.complete(dependency.apply(request))); 14 | return result; 15 | }; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/topology/AsyncUnaryOperator.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.topology; 2 | 3 | import ai.diffy.functional.algebra.monoids.functions.UnaryOperator; 4 | 5 | import java.util.concurrent.CompletableFuture; 6 | import java.util.function.Function; 7 | 8 | public interface AsyncUnaryOperator extends UnaryOperator> { 9 | @Override 10 | default Function> apply(Function dependency) { 11 | return request -> CompletableFuture.completedFuture(dependency.apply(request)); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/topology/ControlFlowLogger.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.topology; 2 | 3 | import ai.diffy.functional.algebra.UnsafeFunction; 4 | import ai.diffy.functional.endpoints.Endpoint; 5 | import ai.diffy.functional.functions.Try; 6 | import reactor.util.function.Tuple2; 7 | import reactor.util.function.Tuple3; 8 | import reactor.util.function.Tuples; 9 | 10 | import java.util.ArrayList; 11 | import java.util.List; 12 | import java.util.stream.Collectors; 13 | 14 | public class ControlFlowLogger { 15 | private final List> events = new ArrayList<>(); 16 | 17 | public Endpoint mapper(Endpoint e, List d) { 18 | return e.andThenMiddleware((apply) -> ((UnsafeFunction)((request) -> { 19 | Try result = Try.of(() -> apply.apply(request)); 20 | events.add(Tuples.of(e.getName(), request, result)); 21 | return result.get(); 22 | })).suppressThrowable()).withDownstream(d); 23 | } 24 | 25 | @Override 26 | public String toString() { 27 | return events.stream().map(Tuple2::getT1).collect(Collectors.joining("\n")); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/topology/InvocationLogger.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.topology; 2 | 3 | import ai.diffy.functional.endpoints.Endpoint; 4 | 5 | import java.util.List; 6 | import java.util.concurrent.CompletableFuture; 7 | import java.util.concurrent.atomic.AtomicInteger; 8 | 9 | public class InvocationLogger{ 10 | private static final AtomicInteger invocationCount = new AtomicInteger(0); 11 | 12 | public static final Endpoint mapper(Endpoint e, List d) { 13 | return e.andThenMiddleware((apply) -> (request) -> { 14 | String name = e.getName(); 15 | final int invocationInstance = invocationCount.getAndIncrement(); 16 | System.out.println(name + " starting [" + invocationInstance + "]"); 17 | final long start = System.currentTimeMillis(); 18 | Response result = (Response) apply.apply(request); 19 | if (CompletableFuture.class.isAssignableFrom(result.getClass())) { 20 | ((CompletableFuture) result).whenComplete((response, error) -> { 21 | System.out.println(name + " completed async [" + invocationInstance + "] in " + (System.currentTimeMillis() - start)); 22 | }); 23 | } else { 24 | System.out.println(name + " finished sync [" + invocationInstance + "] in " + (System.currentTimeMillis() - start)); 25 | } 26 | return result; 27 | }).withDownstream(d); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/topology/SpanWrapper.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.topology; 2 | 3 | import ai.diffy.functional.endpoints.Endpoint; 4 | import ai.diffy.functional.endpoints.SymmetricDependentEndpoint; 5 | import io.opentelemetry.api.trace.Span; 6 | 7 | public class SpanWrapper extends SymmetricDependentEndpoint { 8 | public SpanWrapper(Endpoint dependency, TriConsumer spanLogger) { 9 | super(dependency.getName()+".span", dependency, applyDependency -> (Request request) -> { 10 | final Span span = Span.current(); 11 | final Response result = applyDependency.apply(request); 12 | spanLogger.accept(request, result, span); 13 | return result; 14 | }); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/functional/topology/TriConsumer.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.topology; 2 | 3 | public interface TriConsumer { 4 | void accept(Request request, Response response, Span span); 5 | } 6 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/interpreter/Lambda.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.interpreter; 2 | 3 | import ai.diffy.functional.algebra.Bijection; 4 | import ai.diffy.functional.algebra.UnsafeFunction; 5 | import com.fasterxml.jackson.databind.ObjectMapper; 6 | import org.graalvm.polyglot.Context; 7 | import org.graalvm.polyglot.Engine; 8 | import org.graalvm.polyglot.Source; 9 | import org.graalvm.polyglot.Value; 10 | 11 | public class Lambda implements UnsafeFunction { 12 | private static ObjectMapper mapper = new ObjectMapper(); 13 | private static ThreadLocal context = 14 | ThreadLocal.withInitial(() -> 15 | Context 16 | .newBuilder("js") 17 | .option("js.ecmascript-version", "2022") 18 | .engine(Engine 19 | .newBuilder() 20 | .option("engine.WarnInterpreterOnly","false") 21 | .build()) 22 | .build() 23 | ); 24 | 25 | private static Source parserSource = Source.create("js", "(x)=>(JSON.parse(x))"); 26 | private static Source stringifySource = Source.create("js", "(x)=>(JSON.stringify(x))"); 27 | 28 | private final UnsafeFunction applier; 29 | 30 | public Lambda(Class clsResponse, String lambda){ 31 | this(clsResponse, Source.create("js", lambda)); 32 | } 33 | public Lambda(Class clsResponse, Source lambda){ 34 | Value parser = context.get().eval(parserSource); 35 | Value stringify = context.get().eval(stringifySource); 36 | Bijection parseJson = Bijection.of(parser::execute, (obj) -> stringify.execute(obj).asString()); 37 | stringify.execute(parser.execute("{}")); // warmup 38 | Value transformation = context.get().eval(lambda); 39 | UnsafeFunction stringifyRequest = mapper::writeValueAsString; 40 | UnsafeFunction parseResponse = str -> mapper.readValue(str, clsResponse); 41 | this.applier = stringifyRequest.andThen(parseJson.wrap(transformation::execute)).andThen(parseResponse); 42 | } 43 | 44 | @Override 45 | public Response apply(Request request) throws Throwable { 46 | return applier.apply(request); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/interpreter/Transformer.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.interpreter; 2 | 3 | import org.graalvm.polyglot.Source; 4 | 5 | public class Transformer extends Lambda { 6 | public Transformer(Class clsResponse, String lambda) { 7 | super(clsResponse, lambda); 8 | } 9 | 10 | public Transformer(Class clsResponse, Source lambda) { 11 | super(clsResponse, lambda); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/interpreter/http/HttpLambdaServer.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.interpreter.http; 2 | 3 | import ai.diffy.interpreter.Lambda; 4 | import ai.diffy.proxy.HttpEndpoint; 5 | import ai.diffy.proxy.HttpMessage; 6 | import ai.diffy.proxy.HttpRequest; 7 | import ai.diffy.proxy.HttpResponse; 8 | import io.netty.handler.codec.http.HttpResponseStatus; 9 | import org.graalvm.polyglot.Source; 10 | import reactor.core.publisher.Mono; 11 | import reactor.netty.DisposableServer; 12 | import reactor.netty.http.server.HttpServer; 13 | 14 | import java.io.File; 15 | import java.io.IOException; 16 | import java.util.function.Function; 17 | 18 | public class HttpLambdaServer { 19 | private final DisposableServer server; 20 | private final Function lambda; 21 | public HttpLambdaServer(int port, String httpLambda) { 22 | this(port, Source.create("js", httpLambda)); 23 | } 24 | public HttpLambdaServer(int port, File httpLambda) throws IOException { 25 | this(port, Source.newBuilder("js", httpLambda).build()); 26 | } 27 | public HttpLambdaServer(int port, Source httpLambda) { 28 | lambda = new Lambda(HttpResponse.class, httpLambda).suppressThrowable(); 29 | server = HttpServer.create() 30 | .port(port) 31 | .httpRequestDecoder(httpRequestDecoderSpec -> 32 | httpRequestDecoderSpec 33 | .maxChunkSize(32*1024*1024) 34 | .maxHeaderSize(32*1024*1024)) 35 | .handle((req, res) -> 36 | Mono.fromFuture( 37 | HttpEndpoint.RequestBuffer 38 | .apply(req) 39 | .thenApply(lambda) 40 | ).flatMap(r -> 41 | res 42 | .status(HttpResponseStatus.parseLine(r.getStatus())) 43 | .headers(HttpMessage.toHttpHeaders(r.getHeaders())) 44 | .sendString(Mono.justOrEmpty(r.getBody())) 45 | .then() 46 | ) 47 | ).bindNow(); 48 | } 49 | 50 | public void shutdown() { 51 | server.disposeNow(); 52 | } 53 | public static void main(String[] args) throws Exception { 54 | HttpLambdaServer primary = new HttpLambdaServer(Integer.parseInt(args[0]), new File("src/main/scala/ai/diffy/interpreter/http/master.js")); 55 | HttpLambdaServer secondary = new HttpLambdaServer(Integer.parseInt(args[1]), new File("src/main/scala/ai/diffy/interpreter/http/master.js")); 56 | HttpLambdaServer candidate = new HttpLambdaServer(Integer.parseInt(args[2]), new File("src/main/scala/ai/diffy/interpreter/http/candidate.js")); 57 | 58 | primary.server.onDispose().block(); 59 | secondary.server.onDispose().block(); 60 | candidate.server.onDispose().block(); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/interpreter/http/candidate.js: -------------------------------------------------------------------------------- 1 | (request) => { 2 | const { 3 | uri, 4 | method, 5 | path, 6 | params, 7 | headers, 8 | body 9 | } = request; 10 | 11 | const response = { 12 | status: '200 OK', 13 | headers: {}, 14 | body: '' 15 | } 16 | 17 | if(uri.startsWith('/api/v2/')){ 18 | response.body = body 19 | } else { 20 | try { 21 | const json = JSON.parse(body) 22 | const result = {} 23 | Object.entries(json).foreach(([key, value]) => { 24 | result[key.toLowerCase()] = value 25 | }) 26 | response.body = result 27 | } catch(e) { 28 | response.body = body 29 | } 30 | } 31 | return response; 32 | } -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/interpreter/http/master.js: -------------------------------------------------------------------------------- 1 | (request) => { 2 | const { 3 | uri, 4 | method, 5 | path, 6 | params, 7 | headers, 8 | body 9 | } = request; 10 | 11 | const response = { 12 | status: '200 OK', 13 | headers: {}, 14 | body: '' 15 | } 16 | 17 | response.body = body; 18 | return response; 19 | } -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/lifter/AnalysisRequest.scala: -------------------------------------------------------------------------------- 1 | package ai.diffy.lifter; 2 | 3 | case class AnalysisRequest( 4 | request: Message, 5 | candidate: Message, 6 | primary: Message, 7 | secondary: Message) 8 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/lifter/FieldMap.scala: -------------------------------------------------------------------------------- 1 | package ai.diffy.lifter 2 | 3 | class FieldMap(val value: Map[String, _]){ 4 | override def toString: String = { 5 | value.toSeq.sortBy { case (k, _) => k }.toString 6 | } 7 | } -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/lifter/HtmlLifter.scala: -------------------------------------------------------------------------------- 1 | package ai.diffy.lifter 2 | 3 | import org.jsoup.Jsoup 4 | import org.jsoup.nodes.{Document, Element} 5 | import org.jsoup.select.Elements 6 | 7 | import scala.collection.JavaConverters._ 8 | import scala.language.postfixOps 9 | 10 | object HtmlLifter { 11 | def lift(node: Element): FieldMap = node match { 12 | case doc: Document => 13 | new FieldMap( 14 | Map( 15 | "head" -> lift(doc.head), 16 | "body" -> lift(doc.body) 17 | ) 18 | ) 19 | case doc: Element => { 20 | val children: Elements = doc.children 21 | val attributes = 22 | new FieldMap( 23 | doc.attributes.asList.asScala map { attribute => 24 | attribute.getKey -> attribute.getValue 25 | } toMap 26 | ) 27 | 28 | new FieldMap( 29 | Map( 30 | "tag" -> doc.tagName, 31 | "text" -> doc.ownText, 32 | "attributes" -> attributes, 33 | "children" -> children.asScala.map(element => lift(element)) 34 | ) 35 | ) 36 | } 37 | } 38 | 39 | def decode(html: String): Document = Jsoup.parse(html) 40 | } -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/lifter/HttpLifter.scala: -------------------------------------------------------------------------------- 1 | package ai.diffy.lifter 2 | 3 | import ai.diffy.Settings 4 | import ai.diffy.proxy.{HttpMessage, HttpRequest, HttpResponse} 5 | import ai.diffy.util.ResourceMatcher 6 | import org.springframework.beans.factory.annotation.Autowired 7 | import org.springframework.stereotype.Component 8 | 9 | import scala.collection.JavaConverters._ 10 | 11 | object HttpLifter { 12 | 13 | 14 | val ControllerEndpointHeaderName = "X-Action-Name" 15 | 16 | def contentTypeNotSupportedException(contentType: String) = new Exception(s"Content type: $contentType is not supported") 17 | 18 | case class MalformedJsonContentException(cause: Throwable) 19 | extends Exception("Malformed Json content") 20 | { 21 | initCause(cause) 22 | } 23 | } 24 | 25 | class HttpLifter(settings: Settings) { 26 | val excludeHttpHeadersComparison: Boolean = settings.excludeHttpHeadersComparison 27 | val resourceMatcher: Option[ResourceMatcher] = settings.resourceMatcher 28 | 29 | import HttpLifter._ 30 | 31 | private[this] def headersMap(response: HttpMessage): Map[String, Any] = { 32 | if(!excludeHttpHeadersComparison) { 33 | Map( "headers" -> new FieldMap(response.getHeaders.asScala.toMap map {case (k,v) => k.toLowerCase ->v })) 34 | } else Map.empty 35 | } 36 | 37 | def liftRequest(req: HttpRequest): Message = { 38 | val headers = req.getHeaders.asScala.toMap 39 | 40 | val canonicalResource: Option[String] = headers 41 | .get("Canonical-Resource") 42 | .orElse(resourceMatcher.flatMap(_.resourceName(req.getPath))) 43 | .orElse(Some(s"${req.getMethod}:${req.getPath}")) 44 | 45 | val params = req.getParams 46 | val body = StringLifter.lift(req.getBody) 47 | Message( 48 | canonicalResource, 49 | new FieldMap( 50 | Map( 51 | "method" -> req.getMethod, 52 | "path" -> req.getPath, 53 | "uri" -> req.getUri, 54 | "headers" -> headers, 55 | "params" -> params, 56 | "body" -> body 57 | ) 58 | ) 59 | ) 60 | } 61 | 62 | def liftResponse(r: HttpResponse): Message = { 63 | val responseMap = Map("status" -> r.getStatus, "body" -> StringLifter.lift(r.getBody())) ++ headersMap(r) 64 | Message(None, new FieldMap(responseMap)) 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/lifter/JsonLifter.scala: -------------------------------------------------------------------------------- 1 | package ai.diffy.lifter 2 | 3 | import com.fasterxml.jackson.core.{JsonGenerator, JsonToken} 4 | import com.fasterxml.jackson.databind.annotation.JsonSerialize 5 | import com.fasterxml.jackson.databind.module.SimpleModule 6 | import com.fasterxml.jackson.databind.ser.std.StdSerializer 7 | import com.fasterxml.jackson.databind.{JsonNode, JsonSerializer, ObjectMapper, SerializerProvider} 8 | import com.fasterxml.jackson.module.scala.DefaultScalaModule 9 | import org.slf4j.LoggerFactory 10 | 11 | import scala.collection.JavaConverters._ 12 | import scala.language.postfixOps 13 | import scala.util.control.NoStackTrace 14 | 15 | object JsonLifter { 16 | val log = LoggerFactory.getLogger(JsonLifter.getClass) 17 | 18 | @JsonSerialize(using = classOf[JsonNullSerializer]) 19 | object JsonNull 20 | object JsonParseError extends Exception with NoStackTrace 21 | 22 | val Mapper = new ObjectMapper 23 | Mapper.registerModule(DefaultScalaModule) 24 | 25 | class fieldMapSerializer extends JsonSerializer[FieldMap]() { 26 | override def serialize(t: FieldMap, jsonGenerator: JsonGenerator, serializerProvider: SerializerProvider): Unit = { 27 | jsonGenerator.writeObject(t.value) 28 | } 29 | } 30 | val module = new SimpleModule() 31 | module.addSerializer(classOf[FieldMap], new fieldMapSerializer) 32 | Mapper.registerModule(module) 33 | 34 | def apply(obj: Any): JsonNode = Mapper.valueToTree(obj) 35 | 36 | def lift(node: JsonNode): Any = node.asToken match { 37 | case JsonToken.START_ARRAY => 38 | node.elements.asScala.toSeq.map { 39 | element => lift(element) 40 | } 41 | case JsonToken.START_OBJECT => { 42 | val fields = node.fieldNames.asScala.toSet 43 | if (areMapInsteadofObjectKeys(fields)) { 44 | node.fields.asScala map {field => (field.getKey -> lift(field.getValue))} toMap 45 | } else { 46 | new FieldMap( 47 | node.fields.asScala map {field => (field.getKey -> lift(field.getValue))} toMap 48 | ) 49 | } 50 | } 51 | case JsonToken.VALUE_FALSE => false 52 | case JsonToken.VALUE_NULL => JsonNull 53 | case JsonToken.VALUE_NUMBER_FLOAT => node.asDouble 54 | case JsonToken.VALUE_NUMBER_INT => node.asLong 55 | case JsonToken.VALUE_TRUE => true 56 | case JsonToken.VALUE_STRING => node.textValue 57 | case _ => throw JsonParseError 58 | } 59 | 60 | def decode(json: String): JsonNode = Mapper.readTree(json) 61 | def decode[T](json: String, clss: Class[T]) = Mapper.readValue(json, clss) 62 | def encode(item: Any): String = Mapper.writer.writeValueAsString(item) 63 | 64 | def areMapInsteadofObjectKeys(fields: Set[String]): Boolean = 65 | fields.size > 50 || fields.exists{ field => 66 | field.length > 100 || 67 | field.matches("[0-9].*") || // starts with a digit 68 | !field.matches("[_a-zA-Z0-9]*") // contains non-alphanumeric characters 69 | } 70 | 71 | } 72 | 73 | class JsonNullSerializer(clazz: Class[Any]) extends StdSerializer[Any](clazz) { 74 | def this() { 75 | this(null) 76 | } 77 | 78 | override def serialize(t: Any, jsonGenerator: JsonGenerator, serializerProvider: SerializerProvider): Unit = { 79 | jsonGenerator.writeNull() 80 | } 81 | } -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/lifter/Message.scala: -------------------------------------------------------------------------------- 1 | package ai.diffy.lifter 2 | 3 | case class Message(endpoint: Option[String], result: FieldMap) -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/lifter/StringLifter.scala: -------------------------------------------------------------------------------- 1 | package ai.diffy.lifter 2 | 3 | import scala.util.Try 4 | 5 | object StringLifter { 6 | val htmlRegexPattern = """<("[^"]*"|'[^']*'|[^'">])*>""".r 7 | 8 | def lift(string: String): Any = { 9 | if(string == null) null else 10 | Try(new FieldMap(Map("type" -> "json", "value" -> JsonLifter.lift(JsonLifter.decode(string))))).getOrElse { 11 | if(htmlRegexPattern.findFirstIn(string).isDefined) 12 | new FieldMap(Map("type" -> "html", "value" -> HtmlLifter.lift(HtmlLifter.decode(string)))) 13 | else string 14 | } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/metrics/MetricsReceiver.scala: -------------------------------------------------------------------------------- 1 | package ai.diffy.metrics 2 | 3 | import ai.diffy.util.Memoize 4 | import io.micrometer.core.instrument.{Counter, Metrics, Tag, Tags} 5 | 6 | import scala.jdk.CollectionConverters.IterableHasAsJava 7 | import scala.language.postfixOps 8 | 9 | object MetricsReceiver { 10 | private class MemoizedMetricsReceiver(val name: String, val tags: Map[String, String]) extends MetricsReceiver { 11 | 12 | override lazy val counter = Metrics.globalRegistry.counter(name, Tags.of(tags map {case (k, v) => Tag.of(k,v) } asJava)) 13 | 14 | override def withNameToken(token: String): MetricsReceiver = 15 | new MemoizedMetricsReceiver(name + "_" + token, tags) 16 | 17 | override def withAdditionalTags(additionalTags: Map[String, String]): MetricsReceiver = 18 | new MemoizedMetricsReceiver(name, tags ++ additionalTags) 19 | } 20 | 21 | type MetricsConstructionArgs = (String, Map[String, String]) 22 | private def apply: MetricsConstructionArgs => MetricsReceiver = Memoize { args => new MemoizedMetricsReceiver(args._1, args._2) } 23 | def root: MetricsReceiver = apply("diffy", Map.empty[String, String]) 24 | } 25 | 26 | trait MetricsReceiver { 27 | def withNameToken(name: String): MetricsReceiver 28 | def withAdditionalTags(tags: Map[String, String]): MetricsReceiver 29 | def counter: Counter 30 | } -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/proxy/AnalysisSpanLogger.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.proxy; 2 | 3 | import ai.diffy.analysis.DifferenceResult; 4 | import ai.diffy.lifter.AnalysisRequest; 5 | import ai.diffy.functional.topology.TriConsumer; 6 | import io.opentelemetry.api.common.AttributeKey; 7 | import io.opentelemetry.api.common.Attributes; 8 | import io.opentelemetry.api.trace.Span; 9 | import scala.Option; 10 | 11 | public class AnalysisSpanLogger implements TriConsumer, Span> { 12 | private AnalysisSpanLogger(){}; 13 | public static AnalysisSpanLogger INSTANCE = new AnalysisSpanLogger(); 14 | @Override 15 | public void accept(AnalysisRequest analysisRequest, Option result, Span span) { 16 | result.foreach(diffResult -> 17 | span.addEvent("DifferenceResult", Attributes.of( 18 | AttributeKey.stringKey("endpoint"), diffResult.endpoint, 19 | AttributeKey.stringKey("request"), diffResult.request, 20 | AttributeKey.stringKey("candidate"), diffResult.responses.candidate, 21 | AttributeKey.stringKey("primary"), diffResult.responses.primary, 22 | AttributeKey.stringKey("secondary"), diffResult.responses.secondary 23 | )) 24 | ); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/proxy/HttpEndpoint.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.proxy; 2 | 3 | import ai.diffy.BaseUrl; 4 | import ai.diffy.Downstream; 5 | import ai.diffy.HostPort; 6 | import ai.diffy.functional.endpoints.Endpoint; 7 | import ai.diffy.functional.endpoints.IndependentEndpoint; 8 | import ai.diffy.functional.topology.Async; 9 | import ai.diffy.transformations.TransformationEdge; 10 | import io.netty.handler.codec.http.HttpHeaderNames; 11 | import io.netty.handler.codec.http.HttpHeaderValues; 12 | import io.netty.handler.codec.http.HttpMethod; 13 | import reactor.core.publisher.Mono; 14 | import reactor.netty.ByteBufMono; 15 | import reactor.netty.http.client.HttpClient; 16 | import reactor.netty.http.server.HttpServerRequest; 17 | 18 | import java.util.concurrent.CompletableFuture; 19 | import java.util.function.Function; 20 | 21 | public class HttpEndpoint extends IndependentEndpoint { 22 | private static final Function> requestBuffer = (req) -> { 23 | if(req.isMultipart()){ 24 | throw new RuntimeException("Content-Type : multipart/form-data is not supported"); 25 | } 26 | if(req.isFormUrlencoded() && 27 | HttpHeaderValues.APPLICATION_X_WWW_FORM_URLENCODED 28 | .contentEquals(req.requestHeaders().get(HttpHeaderNames.CONTENT_TYPE))){ 29 | throw new RuntimeException("Content-Type : application/x-www-form-urlencoded is not supported"); 30 | } 31 | return req.receive().aggregate().asString().toFuture().thenApply(body -> new HttpRequest( 32 | req.method().name(), 33 | req.uri(), 34 | req.path(), 35 | req.params(), 36 | req.requestHeaders(), 37 | body, 38 | TransformationEdge.all.toString() 39 | )); 40 | }; 41 | 42 | public static final Endpoint> RequestBuffer = 43 | Async.contain(Endpoint.from("RequestBuffer", () -> requestBuffer)); 44 | public HttpEndpoint(String name, HttpClient client) { 45 | super(name, () -> (HttpRequest req) -> 46 | client 47 | .headers(headers -> headers.add(HttpMessage.toHttpHeaders(req.getHeaders()))) 48 | .request(HttpMethod.valueOf(req.getMethod())) 49 | .uri(req.getUri()) 50 | .send(ByteBufMono.fromString(Mono.justOrEmpty(req.getBody()))) 51 | .responseSingle( 52 | (headers, body) -> 53 | body.asString() 54 | .map(b -> new HttpResponse(headers.status().toString(), headers.responseHeaders(), b)) 55 | ).block() 56 | ); 57 | } 58 | public Endpoint> withSeverRequestBuffer(){ 59 | return Endpoint.from(this.getName(), () -> (serverRequest -> requestBuffer.apply(serverRequest).thenApply(this::apply))); 60 | } 61 | private static HttpEndpoint from(String name, String host, int port, int maxHeader) { 62 | final HttpClient client = HttpClient 63 | .create().host(host).port(port) 64 | .httpResponseDecoder(httpResponseDecoderSpec -> 65 | httpResponseDecoderSpec 66 | .maxHeaderSize(maxHeader)); 67 | return new HttpEndpoint(name, client); 68 | } 69 | private static HttpEndpoint from(String name, String baseUrl, int maxHeader) { 70 | final HttpClient client = HttpClient 71 | .create().baseUrl(baseUrl) 72 | .httpResponseDecoder(httpResponseDecoderSpec -> 73 | httpResponseDecoderSpec 74 | .maxHeaderSize(maxHeader)); 75 | return new HttpEndpoint(name, client); 76 | } 77 | 78 | public static HttpEndpoint from(String name, Downstream downstream, int maxHeader) { 79 | if(downstream instanceof BaseUrl){ 80 | return from(name, ((BaseUrl) downstream).baseUrl(), maxHeader); 81 | } 82 | HostPort hostport = (HostPort)downstream; 83 | return from(name, hostport.host(), hostport.port(), maxHeader); 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/proxy/HttpMessage.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.proxy; 2 | 3 | import io.netty.handler.codec.http.EmptyHttpHeaders; 4 | import io.netty.handler.codec.http.HttpHeaders; 5 | 6 | import java.util.*; 7 | import java.util.stream.Collectors; 8 | import java.util.stream.StreamSupport; 9 | 10 | public abstract class HttpMessage { 11 | Map headers; 12 | String body; 13 | 14 | public HttpMessage(){} 15 | public HttpMessage(HttpHeaders headers, String body) { 16 | this.headers = group(headers.entries()); 17 | this.body = body; 18 | } 19 | 20 | public Map getHeaders(){ 21 | return this.headers; 22 | } 23 | private static Map group(Iterable> entries){ 24 | Map> grouped = new TreeMap<>(); 25 | entries.forEach(entry -> { 26 | grouped.putIfAbsent(entry.getKey(), new ArrayList<>()); 27 | grouped.get(entry.getKey()).add(entry.getValue()); 28 | }); 29 | Map values = new HashMap<>(grouped.size()); 30 | grouped.entrySet().forEach(entry -> { 31 | values.put(entry.getKey(), String.join(",",entry.getValue().stream().sorted().toList())); 32 | }); 33 | return values; 34 | } 35 | public String getBody(){ 36 | return body; 37 | } 38 | 39 | public static HttpHeaders toHttpHeaders(Map entries) { 40 | HttpHeaders result = EmptyHttpHeaders.INSTANCE.copy(); 41 | entries.forEach((key, values) -> 42 | Arrays.stream(values.split(",")).forEach(value -> 43 | result.add(key, value) 44 | ) 45 | ); 46 | return result; 47 | } 48 | @Override 49 | public String toString() { 50 | String headers = this.getHeaders().entrySet().stream().map(entry -> entry.getKey() + " : " + entry.getValue()).reduce((e1, e2) -> e1 + "\n" + e2).orElse(""); 51 | return "\n" + headers + "\n" + body; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/proxy/HttpRequest.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.proxy; 2 | 3 | import ai.diffy.transformations.TransformationEdge; 4 | import io.netty.handler.codec.http.HttpHeaders; 5 | 6 | import java.util.Map; 7 | 8 | public class HttpRequest extends HttpMessage { 9 | private String method; 10 | private String uri; 11 | private String path; 12 | private Map params; 13 | 14 | private TransformationEdge routingMode; 15 | 16 | public HttpRequest(){ 17 | super(); 18 | } 19 | public HttpRequest(String method, String uri, String path, Map params, HttpHeaders headers, String body, String routingMode) { 20 | super(headers, body); 21 | this.method = method; 22 | this.uri = uri; 23 | this.path = path; 24 | this.params = params; 25 | this.routingMode = TransformationEdge.valueOf(routingMode); 26 | } 27 | 28 | public String getMethod() { 29 | return method; 30 | } 31 | 32 | public String getUri() { 33 | return uri; 34 | } 35 | 36 | public String getPath() { 37 | return path; 38 | } 39 | 40 | public Map getParams() { 41 | return params; 42 | } 43 | 44 | public TransformationEdge getRoutingMode() { 45 | return routingMode; 46 | } 47 | 48 | @Override 49 | public String toString() { 50 | return "path = "+ path+"\n"+"params =\n"+ params+"\n"+"message =\n"+ super.toString()+"\n"; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/proxy/HttpResponse.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.proxy; 2 | 3 | import io.netty.handler.codec.http.HttpHeaders; 4 | 5 | import java.util.Map; 6 | 7 | public class HttpResponse extends HttpMessage { 8 | private String status; 9 | 10 | public HttpResponse(){} 11 | public HttpResponse(String status, HttpHeaders headers, String body) { 12 | super(headers, body); 13 | this.status = status; 14 | } 15 | 16 | public String getStatus() { 17 | return status; 18 | } 19 | 20 | @Override 21 | public String toString() { 22 | return "\nstatus = "+ status+"\n"+"message =\n"+ super.toString()+"\n"; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/proxy/MulticastProxy.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.proxy; 2 | 3 | import ai.diffy.analysis.DifferenceResult; 4 | import ai.diffy.functional.algebra.monoids.functions.OctaOperator; 5 | import ai.diffy.functional.algebra.monoids.functions.SeptaOperator; 6 | import ai.diffy.functional.functions.Try; 7 | import ai.diffy.lifter.AnalysisRequest; 8 | import ai.diffy.lifter.Message; 9 | import ai.diffy.util.Future; 10 | import io.netty.handler.codec.http.EmptyHttpHeaders; 11 | import io.netty.handler.codec.http.HttpResponseStatus; 12 | import reactor.netty.http.server.HttpServerRequest; 13 | import reactor.util.function.Tuple3; 14 | import reactor.util.function.Tuples; 15 | import scala.Option; 16 | 17 | import java.util.concurrent.CompletableFuture; 18 | 19 | public class MulticastProxy { 20 | private static final CompletableFuture empty = 21 | CompletableFuture.completedFuture(new HttpResponse(HttpResponseStatus.OK.toString(), EmptyHttpHeaders.INSTANCE, "")); 22 | 23 | public static final SeptaOperator, 25 | HttpRequest, CompletableFuture, 26 | HttpRequest, CompletableFuture, 27 | AnalysisRequest, CompletableFuture>, 28 | HttpRequest, Message, 29 | HttpResponse, Message, 30 | Tuple3, CompletableFuture, CompletableFuture>, CompletableFuture, 31 | CompletableFuture> Operator = 32 | ( 33 | primary, 34 | secondary, 35 | candidate, 36 | analyzer, 37 | liftRequest, 38 | liftResponse, 39 | responsePicker 40 | ) -> (HttpRequest request) -> { 41 | switch (request.getRoutingMode()) { 42 | case primary : return primary.apply(request); 43 | case secondary : return secondary.apply(request); 44 | case candidate : return candidate.apply(request); 45 | case none : return empty; 46 | case all : { 47 | 48 | CompletableFuture pr = primary.apply(request); 49 | CompletableFuture cr = Future.getAfter(pr, () -> candidate.apply(request)); 50 | CompletableFuture sr = Future.getAfter(cr, () -> secondary.apply(request)); 51 | 52 | sr.thenApply(msgS -> Try.of(() -> { 53 | Message r = liftRequest.apply(request); 54 | Message c = liftResponse.apply(cr.get()); 55 | Message p = liftResponse.apply(pr.get()); 56 | Message s = liftResponse.apply(sr.get()); 57 | return AnalysisRequest.apply(r, c, p, s); 58 | })).thenApply(tryRequest -> tryRequest.map(analyzer)); 59 | return responsePicker.apply(Tuples.of(pr, cr, sr)); 60 | } 61 | } 62 | return empty; 63 | }; 64 | } 65 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/repository/DifferenceResultRepository.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.repository; 2 | 3 | import ai.diffy.analysis.DifferenceResult; 4 | import org.springframework.data.mongodb.repository.MongoRepository; 5 | import org.springframework.stereotype.Component; 6 | import org.springframework.stereotype.Repository; 7 | 8 | import java.util.List; 9 | 10 | @Component 11 | @Repository 12 | public interface DifferenceResultRepository extends MongoRepository{ 13 | List findByTimestampMsecBetween(Long start, Long end); 14 | } 15 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/repository/Noise.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.repository; 2 | 3 | import org.springframework.data.annotation.Id; 4 | import org.springframework.data.mongodb.core.mapping.Document; 5 | 6 | import java.util.List; 7 | 8 | @Document 9 | public class Noise { 10 | @Id public String endpoint; 11 | public List noisyfields; 12 | 13 | public Noise(String endpoint, List noisyfields) { 14 | this.endpoint = endpoint; 15 | this.noisyfields = noisyfields; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/repository/NoiseRepository.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.repository; 2 | 3 | import org.springframework.data.mongodb.repository.MongoRepository; 4 | import org.springframework.stereotype.Component; 5 | import org.springframework.stereotype.Repository; 6 | 7 | @Component 8 | @Repository 9 | public interface NoiseRepository extends MongoRepository { 10 | } 11 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/transformations/Transformation.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.transformations; 2 | 3 | import org.springframework.data.annotation.Id; 4 | import org.springframework.data.mongodb.core.mapping.Document; 5 | 6 | @Document 7 | public class Transformation { 8 | @Id 9 | public String injectionPoint; 10 | public String transformationJs; 11 | 12 | public Transformation(String injectionPoint, String transformationJs) { 13 | this.injectionPoint = injectionPoint; 14 | this.transformationJs = transformationJs; 15 | } 16 | 17 | public String getTransformationJs() { 18 | return transformationJs; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/transformations/TransformationCachingService.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.transformations; 2 | 3 | import ai.diffy.functional.endpoints.Endpoint; 4 | import ai.diffy.interpreter.Transformer; 5 | import ai.diffy.proxy.HttpRequest; 6 | import org.springframework.beans.factory.annotation.Autowired; 7 | import org.springframework.stereotype.Component; 8 | import org.springframework.stereotype.Service; 9 | 10 | import javax.annotation.PostConstruct; 11 | import java.util.Arrays; 12 | import java.util.Objects; 13 | import java.util.Optional; 14 | import java.util.concurrent.ConcurrentHashMap; 15 | 16 | @Component 17 | @Service 18 | public class TransformationCachingService { 19 | @Autowired 20 | TransformationRepository repository; 21 | 22 | private final ConcurrentHashMap transformations = new ConcurrentHashMap<>(TransformationEdge.values().length); 23 | private final ConcurrentHashMap> rxTx = new ConcurrentHashMap<>(TransformationEdge.values().length); 24 | 25 | @PostConstruct 26 | public void postConstruct() { 27 | Arrays.stream(TransformationEdge.values()).flatMap(edge -> 28 | repository.findById(edge.name()).stream() 29 | ).forEach(this::putCacheOnly); 30 | } 31 | private void putCacheOnly(Transformation tx){ 32 | TransformationEdge edge = TransformationEdge.valueOf(tx.injectionPoint); 33 | transformations.put(edge, tx.transformationJs); 34 | rxTx.put(edge, new Transformer<>(HttpRequest.class, tx.transformationJs)); 35 | } 36 | 37 | // Read back 38 | public Optional get(String injectionPoint){ 39 | return Optional.ofNullable(transformations.get(TransformationEdge.valueOf(injectionPoint))); 40 | } 41 | public Optional> get(TransformationEdge edge){ 42 | return Optional.ofNullable(rxTx.get(edge)); 43 | } 44 | 45 | // Write through 46 | public void set(Transformation transformation) { 47 | this.putCacheOnly(transformation); 48 | repository.save(transformation); 49 | } 50 | 51 | public void delete(String injectionPoint){ 52 | transformations.remove(TransformationEdge.valueOf(injectionPoint)); 53 | rxTx.remove(TransformationEdge.valueOf(injectionPoint)); 54 | repository.deleteById(injectionPoint); 55 | } 56 | 57 | // Application for syntactic sugar 58 | public Endpoint apply(TransformationEdge edge, Endpoint endpoint) { 59 | Optional> tx = get(edge); 60 | if(tx.isEmpty()){ 61 | return endpoint; 62 | } 63 | return endpoint.andThenMiddleware((applier) -> applier.compose(tx.get().suppressThrowable())); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/transformations/TransformationController.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.transformations; 2 | 3 | import org.springframework.beans.factory.annotation.Autowired; 4 | import org.springframework.web.bind.annotation.*; 5 | 6 | @RestController 7 | public class TransformationController { 8 | @Autowired 9 | TransformationCachingService service; 10 | 11 | @PostMapping("/api/1/transformations/{injectionPoint}") 12 | public void set(@PathVariable("injectionPoint") String injectionPoint, @RequestBody String transformationJs) { 13 | service.set(new Transformation(injectionPoint, transformationJs)); 14 | } 15 | 16 | @GetMapping("/api/1/transformations/{injectionPoint}") 17 | public Transformation get(@PathVariable("injectionPoint") String injectionPoint) { 18 | return service 19 | .get(injectionPoint) 20 | .map(txJs -> new Transformation(injectionPoint, txJs)) 21 | .orElse(new Transformation(injectionPoint, "(request) => (request)")); 22 | } 23 | 24 | @DeleteMapping("/api/1/transformations/{injectionPoint}") 25 | public void delete(@PathVariable("injectionPoint") String injectionPoint) { 26 | service.delete(injectionPoint); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/transformations/TransformationEdge.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.transformations; 2 | 3 | public enum TransformationEdge { 4 | all, 5 | primary, 6 | secondary, 7 | candidate, 8 | none 9 | } 10 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/transformations/TransformationRepository.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.transformations; 2 | 3 | import org.springframework.data.mongodb.repository.MongoRepository; 4 | import org.springframework.stereotype.Component; 5 | import org.springframework.stereotype.Repository; 6 | 7 | @Component 8 | @Repository 9 | public interface TransformationRepository extends MongoRepository {} 10 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/util/Future.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.util; 2 | 3 | import java.util.concurrent.CompletableFuture; 4 | import java.util.function.Supplier; 5 | 6 | public class Future { 7 | public static void assign(CompletableFuture original, CompletableFuture incomplete) { 8 | original.whenComplete((response, error) -> { 9 | if(error != null) { 10 | incomplete.completeExceptionally(error); 11 | } else { 12 | incomplete.complete(response); 13 | } 14 | }); 15 | } 16 | 17 | public static CompletableFuture getAfter(CompletableFuture blocker, Supplier> main){ 18 | return blocker 19 | .thenCompose((success) -> main.get()) 20 | .exceptionallyCompose((error) -> main.get()); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/util/Memoize.scala: -------------------------------------------------------------------------------- 1 | package ai.diffy.util 2 | 3 | import java.util.concurrent.ConcurrentHashMap 4 | 5 | object Memoize { 6 | def apply[A, B](function: A => B): A => B = { 7 | val map = new ConcurrentHashMap[A, B]() 8 | (a: A) => { 9 | map.computeIfAbsent(a, function.apply) 10 | map.get(a) 11 | } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/util/ResourceMatcher.scala: -------------------------------------------------------------------------------- 1 | package ai.diffy.util 2 | 3 | import ai.diffy.util.ResourceMatcher.ResourceMapping 4 | 5 | import scala.util.matching.Regex 6 | 7 | object ResourceMatcher { 8 | 9 | /** 10 | * _1 - The pattern for the resource 11 | * _2 - The name of the resource 12 | * _3 - A function that returns true if a string matches the pattern 13 | */ 14 | type ResourceMapping = (String, String) 15 | 16 | type PathMatcher = (String, String) => Boolean 17 | } 18 | 19 | class ResourceMatcher private(tokenPattern: Regex, wildcardPattern: Regex, mappings: List[ResourceMapping]) { 20 | 21 | def this(mappings: List[ResourceMapping]) = this(""":\w+""".r, """\*+""".r, mappings) 22 | 23 | private val patterns = mappings 24 | .map { case (pattern, name) => (pattern 25 | .replaceAll(tokenPattern.regex, "\\\\w+") 26 | .replaceAll(wildcardPattern.regex, ".*"), name) 27 | } 28 | 29 | def resourceName(path: String): Option[String] = { 30 | patterns.find { case (regex, _) => path.matches(regex) }.map(_._2) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/scala/ai/diffy/util/ResponseMode.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.util; 2 | 3 | public enum ResponseMode { 4 | primary, 5 | secondary, 6 | candidate, 7 | none 8 | } 9 | -------------------------------------------------------------------------------- /src/test/resources/application.yml: -------------------------------------------------------------------------------- 1 | spring.application.name: "diffy" 2 | server: 3 | port: ${http.port:8888} 4 | 5 | proxy: 6 | port: 8880 7 | 8 | candidate: "localhost:9000" 9 | 10 | master: 11 | primary: "localhost:9100" 12 | secondary: "localhost:9200" 13 | 14 | service: 15 | protocol : "http" 16 | 17 | serviceName : "Default Sample Service" 18 | 19 | allowHttpSideEffects: true 20 | maxHeaderSize : 33554432 21 | 22 | spring: 23 | mongodb: 24 | embedded: 25 | version: "5.0.6" 26 | management: 27 | endpoints: 28 | web: 29 | exposure: 30 | include: health,info,prometheus 31 | 32 | logging: 33 | pattern: 34 | console: "%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(%5p) %clr([diffy,%X{trace_id},%X{span_id}]) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m %n%wEx" 35 | file: "%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(%5p) %clr([diffy,%X{trace_id},%X{span_id}]) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m %n%wEx" -------------------------------------------------------------------------------- /src/test/resources/echo.js: -------------------------------------------------------------------------------- 1 | (request) => { 2 | const { 3 | uri, 4 | method, 5 | path, 6 | params, 7 | headers, 8 | body 9 | } = request; 10 | 11 | delete headers['Content-Length'] 12 | return { 13 | status: '200 OK', 14 | headers, 15 | body 16 | } 17 | } -------------------------------------------------------------------------------- /src/test/resources/lambda.js: -------------------------------------------------------------------------------- 1 | (request) => { 2 | const { 3 | uri, 4 | method, 5 | path, 6 | params, 7 | headers, 8 | body 9 | } = request; 10 | 11 | const response = { 12 | status: '200 OK', 13 | headers: {}, 14 | body: { 15 | "message": "Success", 16 | "response": { 17 | "id": 98758734857, 18 | "app": 12, 19 | "resource": { 20 | "id": 150, 21 | "app": null, 22 | "label": "Test label", 23 | "hfu": { 24 | "id": 898, 25 | "app": null, 26 | "createdDate": null, 27 | "lastModifiedDate": null, 28 | "name": "test_name", 29 | "uuid": null 30 | }, 31 | "custom": false, 32 | "delhuf": false 33 | }, 34 | "title": "Test title from candidate server", 35 | "abcd": "text", 36 | "related": [ 37 | "ioi" 38 | ], 39 | "data": null, 40 | "flabel": "Test", 41 | "tcon": null, 42 | "type": "UYU", 43 | "gb": [], 44 | "meas": null, 45 | "audy": null, 46 | "stpe": null, 47 | "slim": 0, 48 | "zoomInData": null, 49 | "improvement": null, 50 | "isdterange": true, 51 | "meta": null, 52 | "intcof": null, 53 | "metqu": null, 54 | "fif": { 55 | "id": null, 56 | "requestResource": null, 57 | "resource": null, 58 | "updatable": false, 59 | "empty": true 60 | }, 61 | "modified": false, 62 | "description": null, 63 | "initdat": null, 64 | "applyModifiedData": false, 65 | "sortConfiguration": null, 66 | "pasfor": null, 67 | "originalTitle": "Test original title", 68 | "uuid": "345345346", 69 | "cofn": null, 70 | "rol": null, 71 | "accessible": true, 72 | "hasDel": false, 73 | "witype": null, 74 | "empwi": false, 75 | "syd": null, 76 | "asgr": [], 77 | "ismod": false, 78 | "ughuygegg": false, 79 | "deleted": false, 80 | "newef": false, 81 | "efor": null 82 | }, 83 | "path": null 84 | } 85 | } 86 | 87 | response.body = body; 88 | return response; 89 | } -------------------------------------------------------------------------------- /src/test/resources/payload.json: -------------------------------------------------------------------------------- 1 | { 2 | "message": "Success", 3 | "response": { 4 | "id": 98758734857, 5 | "app": 12, 6 | "resource": { 7 | "id": 150, 8 | "app": null, 9 | "label": "Test label", 10 | "hfu": { 11 | "id": 898, 12 | "app": null, 13 | "createdDate": null, 14 | "lastModifiedDate": null, 15 | "name": "test_name", 16 | "uuid": null 17 | }, 18 | "custom": false, 19 | "delhuf": false 20 | }, 21 | "title": "Test title from candidate server", 22 | "abcd": "text", 23 | "related": [ 24 | "ioi" 25 | ], 26 | "data": null, 27 | "flabel": "Test", 28 | "tcon": null, 29 | "type": "UYU", 30 | "gb": [], 31 | "meas": null, 32 | "audy": null, 33 | "stpe": null, 34 | "slim": 0, 35 | "zoomInData": null, 36 | "improvement": null, 37 | "isdterange": true, 38 | "meta": null, 39 | "intcof": null, 40 | "metqu": null, 41 | "fif": { 42 | "id": null, 43 | "requestResource": null, 44 | "resource": null, 45 | "updatable": false, 46 | "empty": true 47 | }, 48 | "modified": false, 49 | "description": null, 50 | "initdat": null, 51 | "applyModifiedData": false, 52 | "sortConfiguration": null, 53 | "pasfor": null, 54 | "originalTitle": "Test original title", 55 | "uuid": "345345346", 56 | "cofn": null, 57 | "rol": null, 58 | "accessible": true, 59 | "hasDel": false, 60 | "witype": null, 61 | "empwi": false, 62 | "syd": null, 63 | "asgr": [], 64 | "ismod": false, 65 | "ughuygegg": false, 66 | "deleted": false, 67 | "newef": false, 68 | "efor": null 69 | }, 70 | "path": null 71 | } -------------------------------------------------------------------------------- /src/test/scala/ai/diffy/HttpHeaderValuesTest.java: -------------------------------------------------------------------------------- 1 | package ai.diffy; 2 | 3 | import io.netty.handler.codec.http.HttpHeaderValues; 4 | import org.junit.jupiter.api.Test; 5 | import static org.junit.jupiter.api.Assertions.assertTrue; 6 | 7 | public class HttpHeaderValuesTest { 8 | @Test 9 | public void urlEncodedHeader() { 10 | assertTrue(HttpHeaderValues.APPLICATION_X_WWW_FORM_URLENCODED.contentEquals("application/x-www-form-urlencoded")); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/test/scala/ai/diffy/IntegrationTest.java: -------------------------------------------------------------------------------- 1 | package ai.diffy; 2 | 3 | import ai.diffy.interpreter.http.HttpLambdaServer; 4 | import ai.diffy.proxy.ReactorHttpDifferenceProxy; 5 | import lombok.extern.slf4j.Slf4j; 6 | import org.junit.jupiter.api.AfterAll; 7 | import org.junit.jupiter.api.BeforeAll; 8 | import org.junit.jupiter.api.Test; 9 | import org.junit.jupiter.api.TestInstance; 10 | import org.springframework.beans.factory.annotation.Autowired; 11 | import org.springframework.boot.test.context.SpringBootTest; 12 | import org.springframework.core.io.FileSystemResource; 13 | import org.springframework.http.HttpEntity; 14 | import org.springframework.http.HttpHeaders; 15 | import org.springframework.http.MediaType; 16 | import org.springframework.http.ResponseEntity; 17 | import org.springframework.util.FileCopyUtils; 18 | import org.springframework.web.client.RestTemplate; 19 | import static org.junit.jupiter.api.Assertions.*; 20 | 21 | import java.io.File; 22 | import java.io.IOException; 23 | import java.io.InputStreamReader; 24 | 25 | @Slf4j 26 | @SpringBootTest 27 | @TestInstance(TestInstance.Lifecycle.PER_CLASS) 28 | public class IntegrationTest { 29 | RestTemplate restTemplate = new RestTemplate(); 30 | @Autowired 31 | Settings settings; 32 | 33 | @Autowired 34 | ReactorHttpDifferenceProxy proxy; 35 | private Integer extractPort(Downstream downstream) { 36 | HostPort hostPort = (HostPort) downstream; 37 | return hostPort.port(); 38 | } 39 | HttpLambdaServer primary, secondary, candidate; 40 | String proxyUrl; 41 | @BeforeAll 42 | public void setup() throws IOException { 43 | primary = new HttpLambdaServer(extractPort(settings.primary()), new File("src/test/resources/echo.js")); 44 | secondary = new HttpLambdaServer(extractPort(settings.secondary()), new File("src/test/resources/echo.js")); 45 | candidate = new HttpLambdaServer(extractPort(settings.candidate()), new File("src/test/resources/echo.js")); 46 | proxyUrl = "http://localhost:"+settings.servicePort()+"/base"; 47 | } 48 | 49 | @AfterAll 50 | public void shutdown() { 51 | primary.shutdown(); 52 | secondary.shutdown(); 53 | candidate.shutdown(); 54 | } 55 | 56 | @Test 57 | public void warmup() throws Exception { 58 | HttpHeaders headers = new HttpHeaders(); 59 | headers.setContentType(MediaType.APPLICATION_JSON); 60 | 61 | FileSystemResource payload = new FileSystemResource("src/test/resources/payload.json"); 62 | String json = FileCopyUtils.copyToString(new InputStreamReader(payload.getInputStream())); 63 | String response = restTemplate.postForObject(proxyUrl, new HttpEntity<>(json, headers), String.class); 64 | assertEquals(json, response); 65 | } 66 | 67 | @Test 68 | public void largeRequestBody() { 69 | int largeSize = 16*1024*1024; // 16 MB 70 | String json = "{\"a\":\""+new String(new char[largeSize])+"\"}"; 71 | log.info("Testing request body of {} MB", json.getBytes().length/1024/1024); 72 | HttpHeaders headers = new HttpHeaders(); 73 | headers.setContentType(MediaType.APPLICATION_JSON); 74 | 75 | String response = restTemplate.postForObject(proxyUrl, new HttpEntity<>(json, headers), String.class); 76 | assertEquals(json, response); 77 | } 78 | 79 | @Test 80 | public void largeRequestHeaders() { 81 | int largeSize = 16*1024*1024; // 16 MB 82 | String json = "{\"a\":\"\"}"; 83 | String header = new String(new char[largeSize]).replaceAll(".", "0"); 84 | log.info("Testing request header of {} MB", header.getBytes().length/1024/1024); 85 | HttpHeaders headers = new HttpHeaders(); 86 | headers.setContentType(MediaType.APPLICATION_JSON); 87 | headers.set("a", header); 88 | 89 | ResponseEntity response = restTemplate.postForEntity(proxyUrl, new HttpEntity<>(json, headers), String.class); 90 | assertEquals(json, response.getBody()); 91 | assertEquals(header, response.getHeaders().getFirst("a")); 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/test/scala/ai/diffy/LoadGenerator.java: -------------------------------------------------------------------------------- 1 | package ai.diffy; 2 | 3 | import ch.qos.logback.classic.Level; 4 | import ch.qos.logback.classic.Logger; 5 | import io.netty.handler.codec.http.HttpHeaderNames; 6 | import io.netty.handler.codec.http.HttpHeaderValues; 7 | import org.slf4j.LoggerFactory; 8 | import reactor.netty.http.client.HttpClient; 9 | 10 | import java.util.stream.IntStream; 11 | 12 | public class LoadGenerator { 13 | public static void main(String[] args) { 14 | Logger root = (Logger)LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME); 15 | root.setLevel(Level.INFO); 16 | int workload = 1024/128; 17 | int concurrency = 8; 18 | int sequence = workload/concurrency; 19 | long begin = System.currentTimeMillis(); 20 | long cpuTime = IntStream.range(0, concurrency).parallel().mapToLong(i -> { 21 | HttpClient client = HttpClient.create().port(8880); 22 | long start = System.currentTimeMillis(); 23 | IntStream.range(0, sequence).forEach(request -> 24 | client.headers(headers -> headers.add(HttpHeaderNames.CONTENT_TYPE, HttpHeaderValues.APPLICATION_JSON)) 25 | .get()// Specifies that POST method will be used 26 | .uri("/json?Mixpanel") // Specifies the path 27 | .responseContent() // Receives the response body 28 | .aggregate() 29 | .asString() 30 | .block() 31 | ); 32 | long duration = System.currentTimeMillis() - start; 33 | System.out.println("loadgen[" + i + "] finished in " + duration + " ms"); 34 | return duration; 35 | }).reduce(0, (a,b)-> a+b); 36 | long clockTime = System.currentTimeMillis() - begin; 37 | System.out.println("Total requests = "+ workload); 38 | System.out.println("Concurrency level = "+ concurrency); 39 | System.out.println("Total clock time = "+ clockTime + " ms"); 40 | System.out.println("Total cpu time = "+ cpuTime + " ms"); 41 | System.out.println("Average cpu time per request = " + 1.0*cpuTime/workload + " ms"); 42 | System.out.println("Throughput = " + (1000 * workload) / clockTime + " rps"); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/test/scala/ai/diffy/compare/DifferenceTest.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.compare; 2 | 3 | import lombok.extern.slf4j.Slf4j; 4 | import org.junit.jupiter.api.Test; 5 | import org.springframework.core.io.FileSystemResource; 6 | import org.springframework.util.FileCopyUtils; 7 | 8 | import java.io.IOException; 9 | import java.io.InputStreamReader; 10 | 11 | @Slf4j 12 | public class DifferenceTest { 13 | 14 | @Test 15 | public void payloadTest() throws IOException { 16 | FileSystemResource payload = new FileSystemResource("src/test/resources/payload.json"); 17 | String json = FileCopyUtils.copyToString(new InputStreamReader(payload.getInputStream())); 18 | log.info(json); 19 | Difference diff = Difference.apply(json, json); 20 | diff.flattened().foreach(tuple -> { 21 | log.info("{} - {}", tuple._1(), tuple._2()); 22 | return null; 23 | }); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/test/scala/ai/diffy/functional/algebra/BijectionTest.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.functional.algebra; 2 | 3 | public class BijectionTest { 4 | public static void main(String[] args) { 5 | Bijection identity = Bijection.of(String::toUpperCase, String::toLowerCase); 6 | try { 7 | identity.apply("s"); 8 | identity.unapply("s"); 9 | String x = identity.wrap(str -> str+str).apply("lower"); 10 | System.out.println(x); 11 | } catch (Throwable e) { 12 | throw new RuntimeException(e); 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /src/test/scala/ai/diffy/interpreter/SimpleTransformerTest.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.interpreter; 2 | 3 | public class SimpleTransformerTest { 4 | public static class Name{ 5 | public String first; 6 | public String last; 7 | } 8 | public static void main(String[] args) { 9 | try { 10 | Name name = new Name(); 11 | name.first = "Puneet"; 12 | name.last = "Khanduri"; 13 | Transformer transformer = new Transformer<>(Name.class, "(name)=>{console.log(JSON.stringify(name,null,4)); return name;}"); 14 | transformer.apply(name); 15 | } catch (Throwable e) { 16 | throw new RuntimeException(e); 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/test/scala/ai/diffy/interpreter/TransformerTest.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.interpreter; 2 | 3 | import ai.diffy.proxy.HttpRequest; 4 | import ai.diffy.transformations.TransformationEdge; 5 | import io.netty.handler.codec.http.EmptyHttpHeaders; 6 | import io.netty.handler.codec.http.HttpHeaders; 7 | import org.graalvm.polyglot.Source; 8 | 9 | import java.io.File; 10 | import java.util.HashMap; 11 | import java.util.Map; 12 | 13 | public class TransformerTest { 14 | public static void main(String [] args) { 15 | try { 16 | Source tx = Source.newBuilder("js", new File("src/test/scala/ai/diffy/interpreter/transform.js")).build(); 17 | Transformer reqTx = new Transformer<>(HttpRequest.class, tx); 18 | Map params = new HashMap<>(); 19 | params.put("hello", "world"); 20 | HttpHeaders headers = EmptyHttpHeaders.INSTANCE; 21 | HttpRequest request = new HttpRequest("POST", "/json?Mixpanel", "/json",params,headers,"oh yeah!", TransformationEdge.all.toString()); 22 | HttpRequest transformed = reqTx.apply(request); 23 | System.out.println("All done!"); 24 | System.out.println("\n\nHere's the original:\n"+request); 25 | System.out.println("\n\nHere's the transformed:\n"+transformed); 26 | } catch (Throwable e) { 27 | throw new RuntimeException(e); 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/test/scala/ai/diffy/interpreter/transform.js: -------------------------------------------------------------------------------- 1 | (request) => { 2 | console.log(request); 3 | let result = request; 4 | result = { 5 | ...result, 6 | headers : { 7 | ...result.headers, 8 | "Content-type": "application/json" 9 | } 10 | }; 11 | return result; 12 | } -------------------------------------------------------------------------------- /src/test/scala/ai/diffy/proxy/ReactorHttpDifferenceProxyTest.java: -------------------------------------------------------------------------------- 1 | package ai.diffy.proxy; 2 | 3 | import reactor.core.publisher.Mono; 4 | 5 | import java.util.concurrent.CompletableFuture; 6 | 7 | public class ReactorHttpDifferenceProxyTest { 8 | public static void main(String[] args) { 9 | Mono.fromFuture(CompletableFuture.failedFuture(new RuntimeException())) 10 | .doOnError(t -> System.out.println("error")) 11 | .block() 12 | ; 13 | } 14 | } 15 | --------------------------------------------------------------------------------