├── .gitignore ├── LICENSE.md ├── README.md ├── building-docker.md ├── docker ├── .env ├── README.md └── compose.yaml ├── k8s └── charts │ ├── arangodb │ ├── Chart.yaml │ ├── templates │ │ ├── ArangoDeployment.yaml │ │ ├── ingress.yaml │ │ └── secrets.yaml │ └── values.yaml │ ├── install-spline-on-k8s.sh │ ├── spline-admin │ ├── Chart.yaml │ ├── templates │ │ └── admin_job.yaml │ └── values.yaml │ ├── spline-ui │ ├── Chart.yaml │ ├── templates │ │ ├── autoscaler.yaml │ │ ├── deployment.yaml │ │ ├── ingress.yaml │ │ └── service.yaml │ └── values.yaml │ └── spline │ ├── Chart.yaml │ ├── templates │ ├── autoscaler.yaml │ ├── deployment.yaml │ ├── ingress.yaml │ └── service.yaml │ └── values.yaml ├── spark-agent-extension-example ├── .gitignore ├── Makefile ├── README.md ├── build.sbt ├── project │ ├── Dependencies.scala │ ├── build.properties │ └── plugins.sbt └── src │ ├── main │ └── scala │ │ └── org │ │ └── example │ │ ├── MyFilter.scala │ │ ├── MyFilterWithParams.scala │ │ ├── MyFilterWithSparkSession.scala │ │ └── MyLineageDispatcher.scala │ └── test │ └── scala │ └── org │ └── example │ ├── MyFilterWithParamsSpec.scala │ └── MyFilterWithSparkSessionSpec.scala ├── spline-on-AWS-demo-setup ├── README.md ├── img.png ├── img_1.png ├── img_2.png ├── img_3.png ├── img_4.png ├── img_5.png ├── img_6.png └── img_7.png ├── spline-on-databricks ├── README.md ├── img.png ├── img_1.png ├── img_10.png ├── img_11.png ├── img_12.png ├── img_2.png ├── img_3.png ├── img_4.png ├── img_5.png ├── img_6.png ├── img_7.png ├── img_8.png └── img_9.png └── spline-testing ├── README.md └── jmetered-spline-testing ├── .env ├── docker-compose.yml ├── lineage-multi.jmx ├── reference ├── chain-lineage-1-2001@50reads-3ops-4attr.json.txt.res ├── chain-lineage-5reads-1-501@50ops-4attr.json.txt.res ├── chain-lineage-5reads-3ops-1-501@50attr.json.txt.res ├── diamond-lineage-1-1001@50reads-3ops-4attr.json.txt.res ├── diamond-lineage-5reads-1-501@50ops-4attr.json.txt.res ├── diamond-lineage-5reads-3ops-1-501@50attr.json.txt.res ├── triangle-lineage-1-501@50reads-3ops-4attr.json.txt.res ├── triangle-lineage-5reads-1-501@50ops-4attr.json.txt.res ├── triangle-lineage-5reads-3-10@10ops-4attr.json.txt.res └── triangle-lineage-5reads-3ops-1-501@50attr.json.txt.res ├── requirements.txt ├── run_jmetered_spline.py ├── tests ├── chain-lineage-1-2001@50reads-3ops-4attr.json.txt ├── chain-lineage-5reads-1-501@50ops-4attr.json.txt ├── chain-lineage-5reads-3ops-1-501@50attr.json.txt ├── diamond-lineage-1-1001@50reads-3ops-4attr.json.txt ├── diamond-lineage-5reads-1-501@50ops-4attr.json.txt ├── diamond-lineage-5reads-3ops-1-501@50attr.json.txt ├── triangle-lineage-1-501@50reads-3ops-4attr.json.txt ├── triangle-lineage-5reads-1-501@50ops-4attr.json.txt ├── triangle-lineage-5reads-3-10@10ops-4attr.json.txt └── triangle-lineage-5reads-3ops-1-501@50attr.json.txt └── user.properties /.gitignore: -------------------------------------------------------------------------------- 1 | /.idea/ 2 | .DS_Store 3 | 4 | /spline-testing/jmetered-spline-testing/spline/ 5 | /spline-testing/jmetered-spline-testing/results/ 6 | /spline-testing/jmetered-spline-testing/processed_results/ 7 | /spline-testing/jmetered-spline-testing/graphs/ 8 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Spline - data lineage tracking solution for data pipelines like Apache Spark and others 2 | 3 | --- 4 | 5 | # Getting started 6 | 7 | The project consists of three main parts: 8 | - [Spark Agent](https://github.com/AbsaOSS/spline-spark-agent) that sits on a driver capturing the data lineage from Spark jobs by analyzing the execution plans 9 | 10 | - [Rest Gateway](https://github.com/AbsaOSS/spline) that receives the lineage data from agent and stores it in the database 11 | 12 | - [Web UI](https://github.com/AbsaOSS/spline-ui) application that visualizes the stored data lineages 13 | 14 | ![Spline diagram](https://user-images.githubusercontent.com/5530211/70050339-fd93f580-15ce-11ea-88b2-4d79ee30d494.png) 15 | 16 | 17 | ## TL;DR 18 | Spin up a Spline server in a Docker 19 | 20 | ```shell 21 | wget https://raw.githubusercontent.com/AbsaOSS/spline-getting-started/main/docker/compose.yaml 22 | 23 | wget https://raw.githubusercontent.com/AbsaOSS/spline-getting-started/main/docker/.env 24 | 25 | SEED=1 docker-compose up 26 | # SEED=1 means to also run sample jobs to populate the database. 27 | ``` 28 | 29 | You can access Spline services on the following URLs: 30 | - Spline Web UI: http://localhost:9090 31 | - Spline Server: http://localhost:8080 32 | 33 | To access Spline UI from another host set `DOCKER_HOST_EXTERNAL` variable pointing to the current host before running `docker-compose`. 34 | Spline UI will propagate it to the user browser so that one will be able to connect to the Spline REST endpoint from the outside of this machine. 35 | 36 | ```shell 37 | DOCKER_HOST_EXTERNAL=192.168.1.222 docker-compose up 38 | ``` 39 | 40 | ## How to extend/customize _Spline Spark Agent_ behavior 41 | 42 | There are three ways how to customize default Spline Spark Agent behavior. Choose the one that fits you needs better. 43 | 44 | 1. A lot of things can be customized declaratively, without any coding needed, by just tweaking 45 | the [Agent configuration](https://github.com/AbsaOSS/spline-spark-agent#configuration). 46 | 2. Spline agent is designed for extension, so the chances are it's enough to override some method or implement some trait to achieve desired behavior, 47 | and attach it as an extension module to your Spark application. See the [example extension project](spark-agent-extension-example). 48 | 3. If the extension API isn't enough then fork the project, replace the Maven coordinates with your custom ones, 49 | and [build](https://github.com/AbsaOSS/spline-spark-agent#building-for-different-scala-and-spark-versions) the agent as your own JAR. 50 | 51 | ## More Howto's 52 | - [Running Spline on Databricks Notebook](spline-on-databricks) 53 | - [Setting up Spline server on AWS](spline-on-AWS-demo-setup) 54 | 55 | --- 56 | 57 | For more information about Spline see - https://absaoss.github.io/spline/ 58 | 59 | Enjoy. 60 | 61 | --- 62 | 63 | Copyright 2019 ABSA Group Limited 64 | 65 | you may not use this file except in compliance with the License. 66 | You may obtain a copy of the License at 67 | 68 | http://www.apache.org/licenses/LICENSE-2.0 69 | 70 | Unless required by applicable law or agreed to in writing, software 71 | distributed under the License is distributed on an "AS IS" BASIS, 72 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 73 | See the License for the specific language governing permissions and 74 | limitations under the License. 75 | 76 | -------------------------------------------------------------------------------- /building-docker.md: -------------------------------------------------------------------------------- 1 | **Note**: All Spline runnable components are available as Docker images on [Docker Hub](https://hub.docker.com/search?q=absaoss%2Fspline&type=image) 2 | 3 | --- 4 | ## How to build Spline Docker images 5 | 6 | The build process is standardized across Spline components. 7 | Every runnable component has a standard _Dockerfile_. 8 | Although you can use a build tool of your choice if you prefer, we recommend to use Apache Maven. 9 | 10 | Our Docker build process leverages [Spotify Dockerfile Maven plugin](https://github.com/spotify/dockerfile-maven) 11 | 12 | One project may have multiple _Dockerfiles_ and hence multiple Docker images will be produced in the build process. 13 | 14 | ### Steps 15 | 16 | 1. Install [Java](https://adoptopenjdk.net/) (version 11 is recommended except for the _Spark Agent_ which strictly requires Java 1.8) 17 | 1. Install [Apache Maven](https://maven.apache.org/) (version 3.6+ is recommended) 18 | 1. Pull the Spline component project that you want to build. 19 | ```shell 20 | git clone https://github.com/AbsaOSS/{spline_repo_of_choice}.git 21 | ``` 22 | 1. From the project root directory, run Maven build command with the `docker` profile enabled, specifying required properties (see below): 23 | ```shell 24 | mvn install \ 25 | -D skipTests \ # Skip unit and integration tests 26 | -D docker \ # Activate "docker" profile 27 | -D dockerfile.repositoryUrl=my # The name prefix of the final Docker image(s) 28 | ``` 29 | Done. 30 | 31 | The above command produces docker images with the name `my/{spline_image_name}` tagged as `{project_version}` and `latest` 32 | 33 | You can verify that docker images were created: 34 | ```shell 35 | docker image ls | grep spline 36 | ``` 37 | 38 | ### Customizing image names and tags 39 | 40 | By default, you have to provide `dockerfile.repositoryUrl` that is used to specify the target repo that is also used as a name prefix for images. 41 | 42 | Instead of `dockerfile.repositoryUrl` you can also use `dockerfile.repositoryPrefix` and optionally `dockerfile.repositorySuffix` 43 | for more precise control on image naming. 44 | For example, the following arguments will create an image with the name `x/y/z/aaa{spline_image_name}bbb` 45 | ```shell 46 | -D dockerfile.repositoryPrefix=x/y/z/aaa 47 | -D dockerfile.repositorySuffix=bbb 48 | ``` 49 | 50 | Similarly, you can add custom prefixes and/or suffixes to the tag names. 51 | For instance, adding the following arguments to the command line would result in tagging the images as `foo_1.2.3_bar` instead of just `1.2.3`: 52 | ```shell 53 | -D dockerfile.versionTagPrefix=foo_ 54 | -D dockerfile.versionTagSuffix=_bar 55 | ``` 56 | 57 | To override tag names completely you can add the following arguments: 58 | ```shell 59 | -D dockerfile.versionTagName=aaa # renames '{project_version}' tag to 'aaa' 60 | -D dockerfile.latestTagName=bbb # renames 'latest' tag to 'bbb' 61 | ``` 62 | 63 | ### Customizing base image names 64 | 65 | There are cases when it's preferable to pull the base images from a custom repo, and sometimes using custom image name prefixes. 66 | 67 | For example, if a _Dockerfile_ is based on `adoptopenjdk/openjdk11:jdk-11.0.10_9-alpine-slim` 68 | and you want to pull it as `foo/bar/baz-adoptopenjdk/openjdk11:jdk-11.0.10_9-alpine-slim` 69 | you can do the following: 70 | 71 | ```shell 72 | -D dockerfile.baseImagePrefix=foo/bar/baz- 73 | ``` 74 | 75 | See https://github.com/AbsaOSS/spline-root-pom/blob/main/pom.xml for details. 76 | 77 | ### Pushing to the Docker repo 78 | 79 | Use `mvn deploy` instead of `mvn install` is you want the images to be also pushed. 80 | 81 | 82 | --- 83 | 84 | Copyright 2019 ABSA Group Limited 85 | 86 | you may not use this file except in compliance with the License. 87 | You may obtain a copy of the License at 88 | 89 | http://www.apache.org/licenses/LICENSE-2.0 90 | 91 | Unless required by applicable law or agreed to in writing, software 92 | distributed under the License is distributed on an "AS IS" BASIS, 93 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 94 | See the License for the specific language governing permissions and 95 | limitations under the License. 96 | -------------------------------------------------------------------------------- /docker/.env: -------------------------------------------------------------------------------- 1 | DOCKER_REPO_URL=absaoss 2 | 3 | # component versions 4 | 5 | ARANGO_DB_VERSION=3.11 6 | 7 | SPLINE_SERVER_VERSION=0.7.9 8 | SPLINE_AGENT_VERSION=2.2.0 9 | SPLINE_UI_VERSION=0.7.6 10 | 11 | SPLINE_SERVER_DOCKER_REPO_URL= 12 | SPLINE_AGENT_DOCKER_REPO_URL= 13 | SPLINE_UI_DOCKER_REPO_URL= 14 | 15 | # exposed ports 16 | 17 | ARANGO_DB_PORT=8530 18 | SPLINE_REST_PORT=8080 19 | SPLINE_UI_PORT=9090 20 | -------------------------------------------------------------------------------- /docker/README.md: -------------------------------------------------------------------------------- 1 | This docker-compose config is designed to start all Spline Docker components in one command. 2 | It's especially useful for trying out and demo purposes. 3 | 4 | ```shell 5 | wget https://raw.githubusercontent.com/AbsaOSS/spline-getting-started/main/docker/compose.yaml 6 | 7 | wget https://raw.githubusercontent.com/AbsaOSS/spline-getting-started/main/docker/.env 8 | 9 | # This command starts all Spline components, leaving the database empty 10 | docker-compose up 11 | 12 | # If you also want to run sample jobs and populate the database with the sample data use this command 13 | SEED=1 docker-compose up 14 | ``` 15 | 16 | You can access Spline services on the following URLs: 17 | - Spline Web UI: http://localhost:9090 18 | - Spline Server: http://localhost:8080 19 | 20 | To access Spline UI from another host set `DOCKER_HOST_EXTERNAL` variable pointing to the current host before running `docker-compose`. 21 | Spline UI will propagate it to the user browser so that one will be able to connect to the Spline REST endpoint from the outside of this machine. 22 | 23 | ```shell 24 | DOCKER_HOST_EXTERNAL=192.168.1.222 docker-compose up 25 | ``` 26 | 27 | For more information about Spline see - https://absaoss.github.io/spline/ 28 | 29 | Enjoy. 30 | 31 | --- 32 | 33 | Copyright 2019 ABSA Group Limited 34 | 35 | you may not use this file except in compliance with the License. 36 | You may obtain a copy of the License at 37 | 38 | http://www.apache.org/licenses/LICENSE-2.0 39 | 40 | Unless required by applicable law or agreed to in writing, software 41 | distributed under the License is distributed on an "AS IS" BASIS, 42 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 43 | See the License for the specific language governing permissions and 44 | limitations under the License. 45 | 46 | -------------------------------------------------------------------------------- /docker/compose.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 ABSA Group Limited 2 | # 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | --- 14 | name: 'spline-demo' 15 | services: 16 | arangodb: 17 | image: arangodb:${ARANGO_DB_VERSION} 18 | restart: on-failure 19 | #entrypoint: arangod --database.auto-upgrade 20 | ports: 21 | - ${ARANGO_DB_PORT}:8529 22 | environment: 23 | ARANGO_NO_AUTH: 1 24 | 25 | rest-server: 26 | image: ${SPLINE_SERVER_DOCKER_REPO_URL:-absaoss}/spline-rest-server:${SPLINE_SERVER_VERSION} 27 | restart: on-failure 28 | ports: 29 | - ${SPLINE_REST_PORT}:8080 30 | environment: 31 | SPLINE_DATABASE_CONNECTION_URL: 'arangodb://arangodb/spline' 32 | # by default /dev/random is used which may block 33 | CATALINA_OPTS: "-Dsecurerandom.source=file:/dev/./urandom -Djava.security.egd=file:/dev/./urandom" 34 | depends_on: 35 | db-init: 36 | condition: service_completed_successfully 37 | 38 | db-init: 39 | image: ${SPLINE_SERVER_DOCKER_REPO_URL:-absaoss}/spline-admin:${SPLINE_SERVER_VERSION} 40 | restart: on-failure 41 | entrypoint: > 42 | tini -g -- bash -c " 43 | until curl --output /dev/null --silent --get --fail http://arangodb:8529/_admin/server/availability 44 | do 45 | echo waiting for ArangoDB server to be ready... 46 | sleep 5 47 | done 48 | exec bash ./entrypoint.sh db-init arangodb://arangodb/spline -s 49 | " 50 | depends_on: 51 | - arangodb 52 | 53 | run-examples: 54 | image: ${SPLINE_AGENT_DOCKER_REPO_URL:-absaoss}/spline-spark-agent:${SPLINE_AGENT_VERSION} 55 | entrypoint: > 56 | tini -g -- bash -c " 57 | if [ ! -z "$SEED" ] 58 | then 59 | until curl --output /dev/null --silent --get --fail http://rest-server:8080 60 | do 61 | echo waiting for Spline server to be ready... 62 | sleep 5 63 | done 64 | exec /entrypoint.sh 65 | fi 66 | " 67 | environment: 68 | SEED: ${SEED} # Controls if the agent examples should run on startup up 69 | SPLINE_PRODUCER_URL: 'http://rest-server:8080/producer' 70 | depends_on: 71 | - rest-server 72 | 73 | ui: 74 | image: ${SPLINE_UI_DOCKER_REPO_URL:-absaoss}/spline-web-ui:${SPLINE_UI_VERSION} 75 | restart: on-failure 76 | environment: 77 | # The consumer URL is used by the web browser 78 | SPLINE_CONSUMER_URL: 'http://${DOCKER_HOST_EXTERNAL:-localhost}:${SPLINE_REST_PORT}/consumer' 79 | # by default /dev/random is used which may block 80 | CATALINA_OPTS: "-Dsecurerandom.source=file:/dev/./urandom -Djava.security.egd=file:/dev/./urandom" 81 | ports: 82 | - ${SPLINE_UI_PORT}:8080 83 | depends_on: 84 | - rest-server 85 | -------------------------------------------------------------------------------- /k8s/charts/arangodb/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | appVersion: "1.0" 3 | description: A Helm chart for Arangodb cluster 4 | name: arangodb-cluster 5 | version: 0.1.0 6 | appVersion: 1.2.6 -------------------------------------------------------------------------------- /k8s/charts/arangodb/templates/ArangoDeployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: "database.arangodb.com/v1alpha" 2 | kind: "ArangoDeployment" 3 | metadata: 4 | name: {{ .Release.Name | quote }} 5 | spec: 6 | mode: {{ .Values.mode }} 7 | environment: {{ .Values.environment }} 8 | externalAccess: 9 | type: "None" 10 | tls: 11 | caSecretName: {{ .Values.tls.caSecretName }} 12 | # ttl: "2160h" 13 | {{- if .Values.annotations }} 14 | annotations: 15 | {{ toYaml .Values.annotations | indent 4 }} 16 | {{- end }} 17 | bootstrap: 18 | passwordSecretNames: 19 | root: {{ .Values.rootPassword }} 20 | agents: 21 | count: {{ .Values.agents.replicasCount }} 22 | args: 23 | - --log.level={{ .Values.agents.logLevel }} 24 | {{- if .Values.agents.persistence.enable }} 25 | resources: 26 | requests: 27 | storage: {{ .Values.agents.persistence.size }} 28 | storageClassName: {{ .Values.agents.persistence.storageClassName }} 29 | {{- end }} 30 | dbservers: 31 | count: {{ .Values.dbservers.replicasCount }} 32 | args: 33 | - --log.level={{ .Values.dbservers.logLevel }} 34 | {{- if .Values.dbservers.persistence.enable }} 35 | resources: 36 | requests: 37 | storage: {{ .Values.dbservers.persistence.size }} 38 | storageClassName: {{ .Values.dbservers.persistence.storageClassName }} 39 | {{- end }} 40 | coordinators: 41 | count: {{ .Values.coordinators.replicasCount }} 42 | args: 43 | - --log.level={{ .Values.coordinators.logLevel }} 44 | image: {{ .Values.repository }}:{{ .Values.tag }} 45 | metrics: 46 | enabled: {{ .Values.metrics.enabled }} 47 | image: {{ .Values.metrics.image }} 48 | mode: {{ .Values.metrics.mode }} 49 | tls: {{ .Values.metrics.tls }} -------------------------------------------------------------------------------- /k8s/charts/arangodb/templates/ingress.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.ingress.enabled }} 2 | apiVersion: networking.k8s.io/v1 3 | kind: Ingress 4 | metadata: 5 | name: {{ .Release.Name }}-ing 6 | labels: 7 | app: {{ .Release.Name }} 8 | release: {{ .Release.Name }} 9 | heritage: {{ .Release.Service }} 10 | {{- with .Values.ingress.annotations }} 11 | annotations: 12 | {{ toYaml . | indent 4 }} 13 | {{- end }} 14 | spec: 15 | #ingressClassName: "nginx" 16 | tls: 17 | - hosts: 18 | - {{ .Values.ingress.host }} 19 | secretName: {{ .Release.Name }}-tls 20 | rules: 21 | - host: {{ .Values.ingress.host }} 22 | http: 23 | paths: 24 | - pathType: Prefix 25 | path: {{ .Values.ingress.path }} 26 | backend: 27 | service: 28 | name: {{ .Release.Name }} 29 | port: 30 | number: {{ .Values.service.port }} 31 | {{- end }} -------------------------------------------------------------------------------- /k8s/charts/arangodb/templates/secrets.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: {{ .Release.Name }}-root-password 5 | labels: 6 | app.kubernetes.io/name: {{ .Release.Name }} 7 | helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} 8 | app.kubernetes.io/managed-by: {{ .Release.Service }} 9 | app.kubernetes.io/instance: {{ .Release.Name }} 10 | release: {{ .Release.Name }} 11 | type: Opaque 12 | data: 13 | username: {{ .Values.auth.username | b64enc | quote }} 14 | password: {{ .Values.auth.password | b64enc | quote }} 15 | -------------------------------------------------------------------------------- /k8s/charts/arangodb/values.yaml: -------------------------------------------------------------------------------- 1 | # mongo-express:0.49.0 2 | registry: "docker.io" 3 | repository: "arangodb" 4 | tag: "3.9.1" 5 | 6 | mode: "Cluster" 7 | environment: "Development" 8 | rootPassword: "Auto" 9 | 10 | auth: 11 | username: "root" 12 | password: "MySuperStrongPassword" 13 | 14 | tls: 15 | caSecretName: "None" 16 | 17 | ## Below applyable only for "mode: Cluster" 18 | agents: 19 | replicasCount: 3 20 | logLevel: "error" 21 | overrideDetectedTotalMemory: true 22 | resources: 23 | limits: 24 | cpu: 500m 25 | memory: 512Mi 26 | requests: 27 | cpu: 100m 28 | memory: 256Mi 29 | persistence: 30 | enable: false 31 | size: "1Gi" 32 | storageClassName: "gp2" 33 | 34 | dbservers: 35 | replicasCount: 3 36 | logLevel: "error" 37 | overrideDetectedTotalMemory: true 38 | resources: 39 | limits: 40 | cpu: 500m 41 | memory: 1Gi 42 | requests: 43 | cpu: 100m 44 | memory: 512Mi 45 | persistence: 46 | enable: false 47 | size: "1Gi" 48 | storageClassName: "gp2" 49 | 50 | coordinators: 51 | replicasCount: 3 52 | logLevel: "error" 53 | overrideDetectedTotalMemory: true 54 | resources: 55 | limits: 56 | cpu: 500m 57 | memory: 512Mi 58 | requests: 59 | cpu: 100m 60 | memory: 512Mi 61 | 62 | metrics: 63 | enabled: false 64 | image: arangodb/arangodb-exporter:0.1.8 65 | mode: "exporter" 66 | tls: false 67 | 68 | service: 69 | port: 8529 70 | 71 | ingress: 72 | enabled: false 73 | host: "arangodb.example.com" 74 | path: "/" 75 | annotations: 76 | external-dns.alpha.kubernetes.io/hostname: "arangodb.example.com" 77 | kubernetes.io/ingress.class: "nginx" 78 | #cert-manager.io/cluster-issuer: cert-manager-issuer -------------------------------------------------------------------------------- /k8s/charts/install-spline-on-k8s.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | NS=spline 3 | API= 4 | UI= 5 | 6 | # API=$(kubectl get svc -n ingress-nginx ingress-nginx-controller -o jsonpath="{.*.loadBalancer.ingress[*].hostname}") 7 | # UI=$(kubectl get svc -n ingress-nginx ingress-nginx-extra-controller -o jsonpath="{.*.loadBalancer.ingress[*].hostname}") 8 | 9 | # ARANGODB-OPERATOR 10 | # https://www.arangodb.com/docs/stable/deployment-kubernetes-usage.html 11 | ARANGO_VERSION=1.2.8 12 | 13 | URLPREFIX=https://github.com/arangodb/kube-arangodb/releases/download/$ARANGO_VERSION 14 | 15 | ## ArangoDB CRD 16 | helm upgrade --install \ 17 | kube-arangodb-crd $URLPREFIX/kube-arangodb-crd-$ARANGO_VERSION.tgz \ 18 | 19 | ## ArangoDB Operator 20 | helm upgrade --install kube-arangodb $URLPREFIX/kube-arangodb-$ARANGO_VERSION.tgz \ 21 | --set operator.replicaCount=1 \ 22 | -n $NS --create-namespace 23 | 24 | # ArangoDB cluster 25 | ARANGO_PASSWORD=MySuperStrongPassword 26 | 27 | helm upgrade --install arangodb arangodb \ 28 | --set auth.password=$ARANGO_PASSWORD \ 29 | --set tls.caSecretName=None \ 30 | -n $NS 31 | 32 | # SPLINE-ADMIN for initdb 33 | helm upgrade --install \ 34 | spline-admin spline-admin \ 35 | --set arango.url="arangodb://root:$ARANGO_PASSWORD@arangodb:8529" \ 36 | -n $NS 37 | 38 | # Spline REST API 39 | helm upgrade --install spline spline \ 40 | --set arango.url="arangodb://root:$ARANGO_PASSWORD@arangodb:8529" \ 41 | --set ingress.enabled="true" \ 42 | --set ingress.host="$API" \ 43 | --set ingress.ingressClass="nginx" \ 44 | -n $NS 45 | 46 | # API=$(kubectl get svc -n $NS spline -o jsonpath="{.*.loadBalancer.ingress[*].hostname}") 47 | 48 | # SPLINE UI 49 | helm upgrade --install spline-ui spline-ui \ 50 | --set splineConsumerUrl="http://$API/consumer" \ 51 | --set ingress.enabled="true" \ 52 | --set ingress.host="$UI" \ 53 | --set ingress.ingressClass="ingress" \ 54 | -n $NS 55 | -------------------------------------------------------------------------------- /k8s/charts/spline-admin/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | description: A Helm chart for Spline-admin 3 | name: spline-admin 4 | version: 0.0.1 5 | appVersion: 0.7.9 -------------------------------------------------------------------------------- /k8s/charts/spline-admin/templates/admin_job.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: batch/v1 2 | kind: Job 3 | metadata: 4 | name: {{ .Release.Name }} 5 | labels: 6 | app: {{ .Release.Name }} 7 | spec: 8 | backoffLimit: 4 9 | completions: 1 10 | # ttlSecondsAfterFinished: 0 11 | activeDeadlineSeconds: 1800 12 | template: 13 | spec: 14 | containers: 15 | - name: initdb 16 | image: {{ .Values.image }}:{{ .Values.imageTag }} 17 | args: 18 | - {{ .Values.cmd }} 19 | - {{ .Values.arango.url }}/{{ .Values.arango.database }} 20 | - {{ .Values.extraArgs }} 21 | resources: 22 | {{- toYaml .Values.resources | nindent 10 }} 23 | restartPolicy: Never -------------------------------------------------------------------------------- /k8s/charts/spline-admin/values.yaml: -------------------------------------------------------------------------------- 1 | arango: 2 | url: "arangodb://root:MySuperStrongPassword@arangodb:8529" 3 | database: "spline" 4 | 5 | cmd: db-init 6 | extraArgs: "-s" 7 | imagePullPolicy: Always 8 | image: "absaoss/spline-admin" 9 | imageTag: "0.7.9" 10 | 11 | resources: -------------------------------------------------------------------------------- /k8s/charts/spline-ui/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | description: A Helm chart for Spline-UI 3 | name: spline-ui 4 | version: 0.0.1 5 | appVersion: 0.7.6 -------------------------------------------------------------------------------- /k8s/charts/spline-ui/templates/autoscaler.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.autoscaling.enabled }} 2 | apiVersion: autoscaling/v1 3 | kind: HorizontalPodAutoscaler 4 | metadata: 5 | name: {{ .Release.Name }} 6 | spec: 7 | minReplicas: {{ .Values.autoscaling.minReplicas }} 8 | maxReplicas: {{ .Values.autoscaling.maxReplicas }} 9 | scaleTargetRef: 10 | apiVersion: apps/v1 11 | kind: Deployment 12 | name: {{ .Release.Name }} 13 | #targetCPUUtilizationPercentage: 70 14 | metrics: 15 | {{- with .Values.autoscaling.memoryAverageUtilization }} 16 | - type: Resource 17 | resource: 18 | name: memory 19 | target: 20 | type: Utilization 21 | averageUtilization: {{ . }} 22 | {{- end }} 23 | {{- with .Values.autoscaling.cpuAverageUtilization }} 24 | - type: Resource 25 | resource: 26 | name: cpu 27 | target: 28 | type: Utilization 29 | averageUtilization: {{ . }} 30 | {{- end }} 31 | {{- end }} -------------------------------------------------------------------------------- /k8s/charts/spline-ui/templates/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: {{ .Release.Name }} 5 | namespace: {{ .Release.Namespace }} 6 | {{- if .Values.annotations }} 7 | annotations: 8 | {{ toYaml .Values.annotations | indent 4 }} 9 | {{- end }} 10 | spec: 11 | selector: 12 | matchLabels: 13 | app: {{ .Release.Name }} 14 | replicas: {{ .Values.replicaCount }} 15 | template: 16 | metadata: 17 | labels: 18 | app: {{ .Release.Name }} 19 | spec: 20 | {{- if .Values.nodeSelector }} 21 | nodeSelector: 22 | {{ toYaml .Values.nodeSelector | indent 8 }} 23 | {{- end }} 24 | containers: 25 | - name: {{ .Release.Name }}-deployment 26 | image: {{ .Values.image }}:{{ .Values.imageTag }} 27 | imagePullPolicy: {{ .Values.imagePullPolicy }} 28 | ports: 29 | - containerPort: {{ .Values.service.port }} 30 | protocol: {{ .Values.service.portType }} 31 | env: 32 | - name: SPLINE_CONSUMER_URL 33 | value: {{ .Values.splineConsumerUrl }} 34 | {{- if .Values.livenessProbe.enabled }} 35 | livenessProbe: 36 | httpGet: 37 | path: {{ .Values.livenessProbe.path }} 38 | port: {{ .Values.service.port }} 39 | initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} 40 | periodSeconds: {{ .Values.livenessProbe.periodSeconds }} 41 | timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} 42 | successThreshold: {{ .Values.livenessProbe.successThreshold }} 43 | failureThreshold: {{ .Values.livenessProbe.failureThreshold }} 44 | {{- end }} 45 | {{- if .Values.readinessProbe.enabled }} 46 | readinessProbe: 47 | httpGet: 48 | path: {{ .Values.readinessProbe.path }} 49 | port: {{ .Values.service.port }} 50 | initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} 51 | periodSeconds: {{ .Values.readinessProbe.periodSeconds }} 52 | timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} 53 | successThreshold: {{ .Values.readinessProbe.successThreshold }} 54 | failureThreshold: {{ .Values.readinessProbe.failureThreshold }} 55 | {{- end }} 56 | resources: 57 | {{- toYaml .Values.resources | nindent 10 }} 58 | -------------------------------------------------------------------------------- /k8s/charts/spline-ui/templates/ingress.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.ingress.enabled }} 2 | apiVersion: networking.k8s.io/v1 3 | kind: Ingress 4 | metadata: 5 | name: {{ .Release.Name }}-ing 6 | labels: 7 | app: {{ .Release.Name }} 8 | release: {{ .Release.Name }} 9 | heritage: {{ .Release.Service }} 10 | {{- if .Values.ingress.annotations }} 11 | annotations: 12 | {{ toYaml .Values.ingress.annotations | indent 4 }} 13 | {{- end }} 14 | spec: 15 | ingressClassName: {{ .Values.ingress.ingressClass }} 16 | {{- if .Values.ingress.tls }} 17 | tls: 18 | {{ toYaml .Values.ingress.tls | indent 4 }} 19 | {{- end }} 20 | rules: 21 | - host: {{ .Values.ingress.host | quote }} 22 | http: 23 | paths: 24 | - pathType: Prefix 25 | path: {{ .Values.ingress.path }} 26 | backend: 27 | service: 28 | name: {{ .Release.Name }} 29 | port: 30 | number: {{ .Values.service.port }} 31 | {{- end }} -------------------------------------------------------------------------------- /k8s/charts/spline-ui/templates/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ .Release.Name }} 5 | {{- if .Values.service.annotations }} 6 | annotations: 7 | {{ toYaml .Values.service.annotations | indent 4 }} 8 | {{- end }} 9 | spec: 10 | type: {{ .Values.service.type }} 11 | selector: 12 | app: {{ .Release.Name }} 13 | ports: 14 | - protocol: {{ .Values.service.portType }} 15 | port: {{ .Values.service.port }} 16 | -------------------------------------------------------------------------------- /k8s/charts/spline-ui/values.yaml: -------------------------------------------------------------------------------- 1 | image: "absaoss/spline-web-ui" 2 | imageTag: "0.7.6" 3 | 4 | #IfNotPresent #Always 5 | imagePullPolicy: IfNotPresent 6 | 7 | replicaCount: 1 8 | 9 | annotations: {} 10 | nodeSelector: {} 11 | 12 | splineConsumerUrl: "http://127.0.0.1:8080/consumer" 13 | 14 | service: 15 | type: "ClusterIP" 16 | port: 8080 17 | portType: TCP 18 | annotations: {} 19 | 20 | ingress: 21 | enabled: false 22 | host: "spline.example.com" 23 | path: "/" 24 | ingressClass: "ingress" 25 | annotations: 26 | # kubernetes.io/ingress.class: "ingress" 27 | tls: 28 | 29 | livenessProbe: 30 | enabled: true 31 | path: "/app/assets/config.json" 32 | initialDelaySeconds: 30 33 | periodSeconds: 10 34 | timeoutSeconds: 5 35 | failureThreshold: 6 36 | successThreshold: 1 37 | 38 | readinessProbe: 39 | enabled: true 40 | path: "/app/assets/config.json" 41 | initialDelaySeconds: 5 42 | periodSeconds: 10 43 | timeoutSeconds: 5 44 | failureThreshold: 6 45 | successThreshold: 1 46 | 47 | resources: 48 | limits: 49 | cpu: 500m 50 | memory: 1Gi 51 | requests: 52 | cpu: 200m 53 | memory: 512Mi 54 | 55 | autoscaling: 56 | enabled: false 57 | minReplicas: 1 58 | maxReplicas: 3 59 | cpuAverageUtilization: 50 60 | memoryAverageUtilization: 70 -------------------------------------------------------------------------------- /k8s/charts/spline/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | description: A Helm chart for Spline 3 | name: spline 4 | version: 0.0.1 5 | appVersion: 0.7.9 -------------------------------------------------------------------------------- /k8s/charts/spline/templates/autoscaler.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.autoscaling.enabled }} 2 | apiVersion: autoscaling/v1 3 | kind: HorizontalPodAutoscaler 4 | metadata: 5 | name: {{ .Release.Name }} 6 | spec: 7 | minReplicas: {{ .Values.autoscaling.minReplicas }} 8 | maxReplicas: {{ .Values.autoscaling.maxReplicas }} 9 | scaleTargetRef: 10 | apiVersion: apps/v1 11 | kind: Deployment 12 | name: {{ .Release.Name }} 13 | #targetCPUUtilizationPercentage: 70 14 | metrics: 15 | {{- with .Values.autoscaling.memoryAverageUtilization }} 16 | - type: Resource 17 | resource: 18 | name: memory 19 | target: 20 | type: Utilization 21 | averageUtilization: {{ . }} 22 | {{- end }} 23 | {{- with .Values.autoscaling.cpuAverageUtilization }} 24 | - type: Resource 25 | resource: 26 | name: cpu 27 | target: 28 | type: Utilization 29 | averageUtilization: {{ . }} 30 | {{- end }} 31 | {{- end }} -------------------------------------------------------------------------------- /k8s/charts/spline/templates/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: {{ .Release.Name }} 5 | annotations: 6 | {{- if .Values.service.annotations }} 7 | {{ toYaml .Values.service.annotations | indent 4 }} 8 | {{- end }} 9 | spec: 10 | selector: 11 | matchLabels: 12 | app: {{ .Release.Name }} 13 | replicas: {{ .Values.replicaCount }} 14 | template: 15 | metadata: 16 | labels: 17 | app: {{ .Release.Name }} 18 | release: {{ .Release.Name }} 19 | spec: 20 | {{- if .Values.nodeSelector }} 21 | nodeSelector: 22 | {{ toYaml .Values.nodeSelector | indent 8 }} 23 | {{- end }} 24 | containers: 25 | - name: {{ .Release.Name }}-deployment 26 | image: {{ .Values.image }}:{{ .Values.imageTag }} 27 | imagePullPolicy: {{ .Values.imagePullPolicy }} 28 | ports: 29 | - containerPort: {{ .Values.service.port }} 30 | protocol: {{ .Values.service.portType }} 31 | env: 32 | - name: spline.database.connectionUrl 33 | value: "{{ .Values.arango.url }}/{{ .Values.arango.database }}" 34 | - name: JAVA_OPTS 35 | value: {{ .Values.JAVA_OPTS }} 36 | {{- if .Values.livenessProbe.enabled }} 37 | livenessProbe: 38 | httpGet: 39 | path: {{ .Values.livenessProbe.path }} 40 | port: {{ .Values.service.port }} 41 | initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} 42 | periodSeconds: {{ .Values.livenessProbe.periodSeconds }} 43 | timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} 44 | successThreshold: {{ .Values.livenessProbe.successThreshold }} 45 | failureThreshold: {{ .Values.livenessProbe.failureThreshold }} 46 | {{- end }} 47 | {{- if .Values.readinessProbe.enabled }} 48 | readinessProbe: 49 | httpGet: 50 | path: {{ .Values.readinessProbe.path }} 51 | port: {{ .Values.service.port }} 52 | initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} 53 | periodSeconds: {{ .Values.readinessProbe.periodSeconds }} 54 | timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} 55 | successThreshold: {{ .Values.readinessProbe.successThreshold }} 56 | failureThreshold: {{ .Values.readinessProbe.failureThreshold }} 57 | {{- end }} 58 | resources: 59 | {{- toYaml .Values.resources | nindent 10 }} -------------------------------------------------------------------------------- /k8s/charts/spline/templates/ingress.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.ingress.enabled }} 2 | apiVersion: networking.k8s.io/v1 3 | kind: Ingress 4 | metadata: 5 | name: {{ .Release.Name }}-ing 6 | labels: 7 | app: {{ .Release.Name }} 8 | release: {{ .Release.Name }} 9 | heritage: {{ .Release.Service }} 10 | {{- if .Values.ingress.annotations }} 11 | annotations: 12 | {{ toYaml .Values.ingress.annotations | indent 4 }} 13 | {{- end }} 14 | spec: 15 | ingressClassName: {{ .Values.ingress.ingressClass }} 16 | {{- if .Values.ingress.tls }} 17 | tls: 18 | {{ toYaml .Values.ingress.tls | indent 4 }} 19 | {{- end }} 20 | rules: 21 | - host: {{ .Values.ingress.host }} 22 | http: 23 | paths: 24 | - pathType: Prefix 25 | path: {{ .Values.ingress.path }} 26 | backend: 27 | service: 28 | name: {{ .Release.Name }} 29 | port: 30 | number: {{ .Values.service.port }} 31 | {{- end }} -------------------------------------------------------------------------------- /k8s/charts/spline/templates/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ .Release.Name }} 5 | {{- if .Values.service.annotations }} 6 | annotations: 7 | {{ toYaml .Values.service.annotations | indent 4 }} 8 | {{- end }} 9 | spec: 10 | type: {{ .Values.service.type }} 11 | selector: 12 | app: {{ .Release.Name }} 13 | ports: 14 | - protocol: {{ .Values.service.portType }} 15 | port: {{ .Values.service.port }} 16 | {{- if eq .Values.service.type "NodePort" }} 17 | nodePort: {{ .Values.service.nodePort }} 18 | targetPort: {{ .Values.service.port }} 19 | {{- end }} 20 | -------------------------------------------------------------------------------- /k8s/charts/spline/values.yaml: -------------------------------------------------------------------------------- 1 | image: "absaoss/spline-rest-server" 2 | imageTag: "0.7.9" 3 | #IfNotPresent #Always 4 | imagePullPolicy: IfNotPresent 5 | 6 | replicaCount: 1 7 | 8 | arango: 9 | url: "arangodb://root:MySuperStrongPassword@arangodb:8529" 10 | database: "spline" 11 | 12 | JAVA_OPTS: "-Xmx512m" 13 | 14 | annotations: {} 15 | nodeSelector: {} 16 | 17 | service: 18 | type: "ClusterIP" 19 | port: 8080 20 | portType: TCP 21 | nodePort: 22 | annotations: {} 23 | 24 | ingress: 25 | enabled: false 26 | host: "spline-api.example.com" 27 | path: "/" 28 | ingressClass: "nginx" 29 | annotations: 30 | # kubernetes.io/ingress.class: "nginx" 31 | tls: 32 | 33 | livenessProbe: 34 | enabled: true 35 | path: "/about/liveness" 36 | initialDelaySeconds: 60 37 | periodSeconds: 10 38 | timeoutSeconds: 5 39 | failureThreshold: 6 40 | successThreshold: 1 41 | 42 | readinessProbe: 43 | enabled: true 44 | path: "/about/readiness" 45 | initialDelaySeconds: 5 46 | periodSeconds: 10 47 | timeoutSeconds: 5 48 | failureThreshold: 6 49 | successThreshold: 1 50 | 51 | resources: 52 | limits: 53 | cpu: 1000m 54 | memory: 2Gi 55 | requests: 56 | cpu: 300m 57 | memory: 1Gi 58 | 59 | autoscaling: 60 | enabled: false 61 | minReplicas: 1 62 | maxReplicas: 5 63 | cpuAverageUtilization: 70 64 | memoryAverageUtilization: 80 -------------------------------------------------------------------------------- /spark-agent-extension-example/.gitignore: -------------------------------------------------------------------------------- 1 | /.idea/ 2 | dist/* 3 | target/ 4 | /.bsp/ 5 | lib_managed/ 6 | src_managed/ 7 | project/boot/ 8 | project/plugins/project/ 9 | .history 10 | .cache 11 | .lib/ 12 | -------------------------------------------------------------------------------- /spark-agent-extension-example/Makefile: -------------------------------------------------------------------------------- 1 | default: clean build 2 | clean: 3 | find . -type d -name "target" | xargs rm -fr 4 | build: 5 | sbt clean +assembly 6 | -------------------------------------------------------------------------------- /spark-agent-extension-example/README.md: -------------------------------------------------------------------------------- 1 | # Spline Spark Agent extension example project 2 | 3 | This project serves as an example and a seed project to build your own extensions 4 | for [Spline Spark Agent](https://github.com/AbsaOSS/spline-spark-agent). 5 | 6 | ## Structure 7 | 8 | The project consist of build files and dummy example source code written in Scala. 9 | 10 | ### `build.sbt` 11 | 12 | Here is where you define all metadata about the project: 13 | 14 | - version 15 | - project name 16 | - organization name 17 | - supported Scala versions 18 | - etc. 19 | 20 | ### `/src/main/scala/` 21 | 22 | Your Spline agent extension source code goes here. The directory contains examples of how to build a 23 | custom [Post Processing Filter](https://github.com/AbsaOSS/spline-spark-agent#filters) 24 | and a custom [Lineage Dispatcher](https://github.com/AbsaOSS/spline-spark-agent#dispatchers). 25 | 26 | Classes `MyFilterWithParams` and `MyFilterWithSparkSession` shows how to pass configuration parameters and a Spark Session respectively to your 27 | custom filter. **The same technique also works for the lineage dispatcher.** 28 | 29 | In order to be instantiated your custom filter or a dispatcher has to define at least one of the following constructors 30 | (listed from the least to the most priority one): 31 | - `this()` – default no-argument constructor 32 | - `this(c: Configuration)` 33 | - `this(c: Configuration, spark: SparkSession)` 34 | 35 | You can choose any of those, and you can combine them with other (overloading) constructors at your convenience. 36 | 37 | ### `/src/test/scala/` 38 | 39 | Test suites 40 | 41 | ## Building 42 | 43 | The project requires [SBT](https://www.scala-sbt.org/) tool for building. 44 | 45 | ### Build final JARs 46 | 47 | ```shell 48 | sbt clean +assembly 49 | ``` 50 | 51 | or for those using linux 52 | 53 | ```shell 54 | make 55 | ``` 56 | 57 | The built JAR-files are located in the `target/scala-$VER/` folder, one per given Scala version. 58 | 59 | ### Clean up 60 | 61 | Remove all generated files and build artifacts 62 | 63 | ```shell 64 | make clean 65 | ``` 66 | 67 | ## Usage 68 | 69 | 1. Add the built JAR-file to the Spark driver classpath using any of available approaches, including but not limited to: 70 | - Copy the JAR-file to the `$SPARK_HOME/jars/` directory 71 | - Include it into the `-jars` param of the `spark-submit`, `spark-shell` or `pyspark` command line 72 | - Unpack the JAR-file and copy its content into your own Spark job fat-JAR, if you have one. 73 | 74 | 2. Register your component in the config. The majority of custom component types (with exception for _Plugin_ trait) require explicit registering in 75 | the [Agent configuration](https://github.com/AbsaOSS/spline-spark-agent#configuration). 76 | See [this section](https://github.com/AbsaOSS/spline-spark-agent#creating-your-own-dispatcher) for example. 77 | 78 | --- 79 | 80 | Copyright 2022 ABSA Group Limited 81 | 82 | you may not use this file except in compliance with the License. 83 | You may obtain a copy of the License at 84 | 85 | http://www.apache.org/licenses/LICENSE-2.0 86 | 87 | Unless required by applicable law or agreed to in writing, software 88 | distributed under the License is distributed on an "AS IS" BASIS, 89 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 90 | See the License for the specific language governing permissions and 91 | limitations under the License. 92 | -------------------------------------------------------------------------------- /spark-agent-extension-example/build.sbt: -------------------------------------------------------------------------------- 1 | import Dependencies.* 2 | 3 | lazy val scala212 = "2.12.15" 4 | lazy val scala211 = "2.11.12" 5 | lazy val supportedScalaVersions = Seq(scala212, scala211) 6 | 7 | ThisBuild / version := "0.1.0-SNAPSHOT" 8 | ThisBuild / organization := "org.example" 9 | ThisBuild / organizationName := "example" 10 | 11 | lazy val root = (project in file(".")) 12 | .settings( 13 | name := "MySplineExtras", 14 | crossScalaVersions := supportedScalaVersions, 15 | assembly / assemblyJarName := s"${name.value}_${scalaBinaryVersion.value}-${version.value}.jar", 16 | assembly / assemblyOption ~= (_.withIncludeScala(false)), 17 | libraryDependencies ++= Seq( 18 | splineAgentCore % Provided, 19 | scalaTest % Test, 20 | scalatestplus % Test, 21 | ) 22 | ) 23 | -------------------------------------------------------------------------------- /spark-agent-extension-example/project/Dependencies.scala: -------------------------------------------------------------------------------- 1 | import sbt.* 2 | 3 | object Dependencies { 4 | lazy val splineAgentCore = "za.co.absa.spline.agent.spark" %% "agent-core" % "2.0.0" 5 | lazy val scalaTest = "org.scalatest" %% "scalatest" % "3.2.15" 6 | lazy val scalatestplus = "org.scalatestplus" %% "mockito-1-10" % "3.1.0.0" 7 | } 8 | -------------------------------------------------------------------------------- /spark-agent-extension-example/project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=1.6.2 2 | -------------------------------------------------------------------------------- /spark-agent-extension-example/project/plugins.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "1.2.0") 2 | -------------------------------------------------------------------------------- /spark-agent-extension-example/src/main/scala/org/example/MyFilter.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.example 18 | 19 | import org.apache.spark.internal.Logging 20 | import za.co.absa.spline.harvester.HarvestingContext 21 | import za.co.absa.spline.harvester.postprocessing.AbstractPostProcessingFilter 22 | import za.co.absa.spline.producer.model._ 23 | 24 | class MyFilter 25 | extends AbstractPostProcessingFilter("My Filter") 26 | with Logging { 27 | 28 | log.info("My custom filter created") 29 | 30 | // Only override and implement methods that you need... 31 | 32 | override def processExecutionEvent(event: ExecutionEvent, ctx: HarvestingContext): ExecutionEvent = event 33 | 34 | override def processExecutionPlan(plan: ExecutionPlan, ctx: HarvestingContext): ExecutionPlan = plan 35 | 36 | override def processReadOperation(op: ReadOperation, ctx: HarvestingContext): ReadOperation = op 37 | 38 | override def processWriteOperation(op: WriteOperation, ctx: HarvestingContext): WriteOperation = op 39 | 40 | override def processDataOperation(op: DataOperation, ctx: HarvestingContext): DataOperation = op 41 | } 42 | -------------------------------------------------------------------------------- /spark-agent-extension-example/src/main/scala/org/example/MyFilterWithParams.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.example 18 | 19 | import org.apache.commons.configuration.Configuration 20 | import org.apache.spark.internal.Logging 21 | import org.example.MyFilterWithParams.ConfProps 22 | import za.co.absa.spline.commons.config.ConfigurationImplicits._ 23 | import za.co.absa.spline.harvester.postprocessing.AbstractPostProcessingFilter 24 | 25 | 26 | object MyFilterWithParams { 27 | object ConfProps { 28 | val Foo = "foo" 29 | val Bar = "bar" 30 | } 31 | } 32 | 33 | class MyFilterWithParams(val foo: String, val bar: Int) 34 | extends AbstractPostProcessingFilter("My Filter With Parameters") 35 | with Logging { 36 | 37 | def this(c: Configuration) = this( 38 | c.getRequiredString(ConfProps.Foo), 39 | c.getRequiredInt(ConfProps.Bar) 40 | ) 41 | 42 | log.info(s"My custom filter created: foo=$foo, bar=$bar") 43 | } 44 | -------------------------------------------------------------------------------- /spark-agent-extension-example/src/main/scala/org/example/MyFilterWithSparkSession.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.example 18 | 19 | import org.apache.commons.configuration.Configuration 20 | import org.apache.spark.internal.Logging 21 | import org.apache.spark.sql.SparkSession 22 | import za.co.absa.spline.harvester.postprocessing.AbstractPostProcessingFilter 23 | 24 | 25 | class MyFilterWithSparkSession(val sparkAppName: String) 26 | extends AbstractPostProcessingFilter("My Filter With SparkSession") 27 | with Logging { 28 | 29 | def this(c: Configuration, sparkSession: SparkSession) = this(sparkSession.sparkContext.appName) 30 | 31 | log.info(s"My custom filter created: Spark application mame = $sparkAppName") 32 | } 33 | -------------------------------------------------------------------------------- /spark-agent-extension-example/src/main/scala/org/example/MyLineageDispatcher.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.example 18 | 19 | import za.co.absa.spline.harvester.dispatcher.LineageDispatcher 20 | import za.co.absa.spline.producer.model.{ExecutionEvent, ExecutionPlan} 21 | 22 | class MyLineageDispatcher extends LineageDispatcher { 23 | 24 | override def send(plan: ExecutionPlan): Unit = ??? 25 | 26 | override def send(event: ExecutionEvent): Unit = ??? 27 | } 28 | -------------------------------------------------------------------------------- /spark-agent-extension-example/src/test/scala/org/example/MyFilterWithParamsSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.example 18 | 19 | import org.apache.commons.configuration.MapConfiguration 20 | import org.scalatest.flatspec.AnyFlatSpec 21 | import org.scalatest.matchers.should.Matchers 22 | import za.co.absa.spline.HierarchicalObjectFactory 23 | import za.co.absa.spline.agent.AgentConfig.ConfProperty 24 | 25 | import scala.collection.JavaConverters._ 26 | 27 | class MyFilterWithParamsSpec extends AnyFlatSpec with Matchers { 28 | "My custom filter" should "load config parameters" in { 29 | val filterName = "myFilterWithParams" 30 | val config = new MapConfiguration(Map( 31 | "spline.postProcessingFilter" -> filterName, 32 | s"spline.postProcessingFilter.$filterName.className" -> classOf[MyFilterWithParams].getName, 33 | s"spline.postProcessingFilter.$filterName.foo" -> "awesome", 34 | s"spline.postProcessingFilter.$filterName.bar" -> "42" 35 | ).asJava) 36 | 37 | val factory = 38 | new HierarchicalObjectFactory(config, null) 39 | .child(ConfProperty.RootPostProcessingFilter) 40 | .child(filterName) 41 | 42 | val myFilter = factory.instantiate[MyFilterWithParams]() 43 | 44 | myFilter.foo should equal("awesome") 45 | myFilter.bar should equal(42) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /spark-agent-extension-example/src/test/scala/org/example/MyFilterWithSparkSessionSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.example 18 | 19 | import org.apache.commons.configuration.MapConfiguration 20 | import org.apache.spark.SparkContext 21 | import org.apache.spark.sql.SparkSession 22 | import org.mockito.Mockito.when 23 | import org.scalatest.flatspec.AnyFlatSpec 24 | import org.scalatest.matchers.should.Matchers 25 | import org.scalatestplus.mockito.MockitoSugar 26 | import za.co.absa.spline.HierarchicalObjectFactory 27 | import za.co.absa.spline.agent.AgentConfig.ConfProperty 28 | 29 | import scala.collection.JavaConverters._ 30 | 31 | class MyFilterWithSparkSessionSpec extends AnyFlatSpec with Matchers with MockitoSugar { 32 | "My custom filter" should "get access to Spark Session" in { 33 | val filterName = "mySparkAwareFilter" 34 | val sparkMock = mock[SparkSession] 35 | val contextMock = mock[SparkContext] 36 | 37 | when(sparkMock.sparkContext) thenReturn contextMock 38 | when(contextMock.appName) thenReturn "My Awesome Application" 39 | 40 | val config = new MapConfiguration(Map( 41 | "spline.postProcessingFilter" -> filterName, 42 | s"spline.postProcessingFilter.$filterName.className" -> classOf[MyFilterWithSparkSession].getName, 43 | s"spline.postProcessingFilter.$filterName.foo" -> "awesome", 44 | s"spline.postProcessingFilter.$filterName.bar" -> "42" 45 | ).asJava) 46 | 47 | val factory = 48 | new HierarchicalObjectFactory(config, sparkMock) 49 | .child(ConfProperty.RootPostProcessingFilter) 50 | .child(filterName) 51 | 52 | val myFilter = factory.instantiate[MyFilterWithSparkSession]() 53 | 54 | myFilter.sparkAppName should equal("My Awesome Application") 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /spline-on-AWS-demo-setup/README.md: -------------------------------------------------------------------------------- 1 | Setting up Spline Server on AWS EC2 2 | === 3 | 4 | Spline is an open-source data lineage tracking tool that can help you to capture data lineage for your various data pipelines. 5 | See [Spline GitHub pages](https://absaoss.github.io/spline/) for details. 6 | 7 | The purpose of this article is to demonstrate the basic steps that need to be done to install Spline Server on AWS EC2. 8 | 9 | ## Disclaimer 10 | 11 | This is **NOT** a production setup guide! 12 | 13 | The approach described below is just enough for demo and trial purposes, but doesn't cover the majority of aspects that need to be considered when 14 | setting up a real production environment. 15 | 16 | ## Prerequisites 17 | 18 | You need to have an AWS account. 19 | 20 | ## Create and launch EC2 instance 21 | 22 | Open your [AWS Console](https://console.aws.amazon.com/) and select **Launch instance** 23 | 24 | ![img.png](img.png) 25 | 26 | ![img_1.png](img_1.png) 27 | 28 | Select an Amazon Machine Image of your choice (we'll use a default Amazon Linux 2 AMI) 29 | 30 | ![img_2.png](img_2.png) 31 | 32 | When choosing an instance type consider the amount of RAM and disk space. You need to run three Docker containers in total: 33 | 34 | - [ArangoDB](https://hub.docker.com/_/arangodb) - this is where the lineage data will be stored. 35 | - [Spline REST Gateway](https://hub.docker.com/r/absaoss/spline-rest-server) - a Java application that exposes an API for Spline agents and the Spline UI. 36 | It runs on a Tomcat server and can be memory intensive. (Alternatively you can use 37 | [Spline Kafka Gateway](https://hub.docker.com/r/absaoss/spline-kafka-server) instead of the REST one, but this is beyond the scope of this article) 38 | - [Spline UI](https://github.com/AbsaOSS/spline-ui) - a lightweight HTTP server that is only used for serving static resources required by the Spline 39 | UI. Spline UI is implemented as a Single Page Application (SAP) that runs entirely within the browser and communicates directly with the Spline Gateway via 40 | the REST API. It does not route any additional HTTP traffic through its own server. 41 | 42 | For demonstration purposes we'll run all three containers on the same EC2 instance, so we'll pick `t2.medium` instance with 4Gb RAM and 2 CPUs. 43 | 44 | ![img_3.png](img_3.png) 45 | 46 | On the **Review Instance** page, check all necessary details. Pay special attention to the security group - the instance needs to be open for 47 | public access. You also need to open two custom TCP ports - one for the REST API and another for the Spline UI. 48 | 49 | ![img_4.png](img_4.png) 50 | 51 | We'll use ports `8080` and `9090` one for the Spline REST API and another the Spline UI. 52 | 53 | Then, we can review and launch our instance. 54 | 55 | ![img_5.png](img_5.png) 56 | 57 | As a final step you'll be asked to create or select a key pair to access the instance via SSH. Follow the AWS instructions. 58 | 59 | ![img_6.png](img_6.png) 60 | 61 | Take a note of the launched instance IP and store it for the rest of the article. 62 | 63 | ![img_7.png](img_7.png) 64 | 65 | ## Setup Spline 66 | 67 | Open the SSH client and log into the instance. 68 | 69 | ```shell 70 | ssh -i ~/.pem/spline-aws.pem ec2-user@18.116.202.35 71 | ``` 72 | 73 | Then install and start the Docker service. 74 | 75 | ```shell 76 | sudo yum install docker -y 77 | sudo systemctl enable docker.service 78 | sudo systemctl start docker.service 79 | sudo usermod -a -G docker ec2-user 80 | ``` 81 | 82 | Re-login to apply the newly added docker group. 83 | 84 | Now we can pull and run Spline containers. You can do it one by one, or use `docker-compose` to run 85 | a [preconfigured demo setup](https://github.com/AbsaOSS/spline-getting-started/tree/main/docker). 86 | 87 | If you want to run individual containers, see the [Step by step instruction](https://absaoss.github.io/spline/#step-by-step). 88 | 89 | For the purpose of this article, we will use Docker Compose. 90 | 91 | Install Docker Compose: 92 | 93 | ```shell 94 | sudo curl -L https://github.com/docker/compose/releases/download/1.21.0/docker-compose-`uname -s`-`uname -m` | sudo tee /usr/local/bin/docker-compose > /dev/null 95 | sudo chmod +x /usr/local/bin/docker-compose 96 | ``` 97 | 98 | Download Spline demo Docker-compose config files: 99 | 100 | ```shell 101 | mkdir spline 102 | cd spline 103 | 104 | wget https://raw.githubusercontent.com/AbsaOSS/spline-getting-started/main/docker/compose.yaml 105 | wget https://raw.githubusercontent.com/AbsaOSS/spline-getting-started/main/docker/.env 106 | ``` 107 | 108 | Run `docker-compose` like below. `DOCKER_HOST_EXTERNAL` is the external IP of this EC2 instance. This IP will be passed to the Spline UI and used by 109 | the client browser to connect to the Spline REST API. 110 | 111 | ```shell 112 | DOCKER_HOST_EXTERNAL=18.116.202.35 docker-compose up 113 | ``` 114 | 115 | The given Docker Compose config also runs a set of Spark examples to pre-populate the database. You can either ignore them, or disable them by 116 | commenting out the `agent` service block in the `compose.yaml` file: 117 | 118 | ```yaml 119 | 120 | # agent: 121 | # image: absaoss/spline-spark-agent:${SPLINE_AGENT_VERSION} 122 | # network_mode: "bridge" 123 | # environment: 124 | # SPLINE_PRODUCER_URL: 'http://172.17.0.1:${SPLINE_REST_PORT}/producer' 125 | # links: 126 | # - spline 127 | 128 | ``` 129 | 130 | When the containers are up we can verify that the Spline Gateway and Spline UI servers are running by visiting the following URLs: 131 | 132 | - http://18.116.202.35:8080/ 133 | - http://18.116.202.35:9090/ 134 | 135 | (Use the correct EC2 instance IP). 136 | 137 | --- 138 | 139 | Copyright 2019 ABSA Group Limited 140 | 141 | you may not use this file except in compliance with the License. 142 | You may obtain a copy of the License at 143 | 144 | http://www.apache.org/licenses/LICENSE-2.0 145 | 146 | Unless required by applicable law or agreed to in writing, software 147 | distributed under the License is distributed on an "AS IS" BASIS, 148 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 149 | See the License for the specific language governing permissions and 150 | limitations under the License. 151 | -------------------------------------------------------------------------------- /spline-on-AWS-demo-setup/img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AbsaOSS/spline-getting-started/96e1187e48746711ec2c582a8bf7f607073bc9e1/spline-on-AWS-demo-setup/img.png -------------------------------------------------------------------------------- /spline-on-AWS-demo-setup/img_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AbsaOSS/spline-getting-started/96e1187e48746711ec2c582a8bf7f607073bc9e1/spline-on-AWS-demo-setup/img_1.png -------------------------------------------------------------------------------- /spline-on-AWS-demo-setup/img_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AbsaOSS/spline-getting-started/96e1187e48746711ec2c582a8bf7f607073bc9e1/spline-on-AWS-demo-setup/img_2.png -------------------------------------------------------------------------------- /spline-on-AWS-demo-setup/img_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AbsaOSS/spline-getting-started/96e1187e48746711ec2c582a8bf7f607073bc9e1/spline-on-AWS-demo-setup/img_3.png -------------------------------------------------------------------------------- /spline-on-AWS-demo-setup/img_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AbsaOSS/spline-getting-started/96e1187e48746711ec2c582a8bf7f607073bc9e1/spline-on-AWS-demo-setup/img_4.png -------------------------------------------------------------------------------- /spline-on-AWS-demo-setup/img_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AbsaOSS/spline-getting-started/96e1187e48746711ec2c582a8bf7f607073bc9e1/spline-on-AWS-demo-setup/img_5.png -------------------------------------------------------------------------------- /spline-on-AWS-demo-setup/img_6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AbsaOSS/spline-getting-started/96e1187e48746711ec2c582a8bf7f607073bc9e1/spline-on-AWS-demo-setup/img_6.png -------------------------------------------------------------------------------- /spline-on-AWS-demo-setup/img_7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AbsaOSS/spline-getting-started/96e1187e48746711ec2c582a8bf7f607073bc9e1/spline-on-AWS-demo-setup/img_7.png -------------------------------------------------------------------------------- /spline-on-databricks/README.md: -------------------------------------------------------------------------------- 1 | Running Spline on Databricks 2 | === 3 | 4 | Spline is an open-source data lineage tracking tool that can help you to capture data lineage of your various data pipelines. 5 | See [Spline GitHub pages](https://absaoss.github.io/spline/) for details. 6 | 7 | In this article, I will demonstrate how to create a minimal Spline set up, and capture data lineage of Spark jobs running on a Databricks Notebook. 8 | 9 | ## Preparation 10 | 11 | ### Install and launch Spline server components 12 | 13 | First, we need to decide where we will run a Spline Gateway. The Spline Gateway is a server part which is responsible for storing and aggregating 14 | lineage metadata captured by Spline agents. It is not strictly required though, as the Spline agent can capture and send the lineage data in Spline 15 | format to any destination, including storing it on S3, HDFS or senfing it to your custom REST API for further processing. However, to fully benefit 16 | from all Spline features (like Spline UI and other features that will come with future Spline versions), the Spline server needs to be installed. 17 | 18 | The simplest way of doing it is using Docker. Since all Spline components are available as Docker, you can run it on any environment that supports 19 | Docker. The only requirement is that Spline REST API should be accessible from the node where Spark driver is executing. For the purpose of this 20 | article we will create a public AWS EC2 instance and run Spline docker containers there. 21 | 22 | See [Spline on AWS - demo setup](../spline-on-AWS-demo-setup/README.md) 23 | Make sure the Spline REST Gateway and the Spline UI servers are running and take note of the _Spline Producer API URL_. It can be found on the Spline 24 | Gateway index page. 25 | 26 | ![img.png](img.png) 27 | 28 | ### Prepare a Databricks account 29 | 30 | You need to have a Databricks account. In this article, we will use a free account 31 | on [Databricks Community Edition](https://community.cloud.databricks.com/login.html) 32 | 33 | ![img_1.png](img_1.png) 34 | 35 | ## Enable Spline on a Databricks cluster 36 | 37 | Create a new cluster on the **Compute** page 38 | 39 | ![img_2.png](img_2.png) 40 | 41 | Pick a desired Databricks runtime and take a note of the selected Spark and Scala versions. Then, go to the **Spark** tab and add required Spline 42 | configuration parameters. 43 | 44 | ![img_4.png](img_4.png) 45 | 46 | Here we instruct Spline agent to use the embedded `http` lineage dispatcher and send the lineage data to out Spline Gateway. Use the _Producer API 47 | URL_ copied in the previous step. 48 | 49 | ```yaml 50 | spark.spline.lineageDispatcher http 51 | spark.spline.lineageDispatcher.http.producer.url http://18.116.202.35:8080/producer 52 | ``` 53 | 54 | You can optionally set the Spline mode to `REQUIRED` if you want Spline pre-flight check errors to be propagated the Spark jobs. It is useful to 55 | minimize the chance for the Spark jobs to complete without capturing lineage, for example due to Spline misconfiguration. 56 | 57 | ```yaml 58 | spark.spline.mode REQUIRED 59 | ``` 60 | 61 | Refer the [Spline agent configuration](https://github.com/AbsaOSS/spline-spark-agent#configuration) section for details about other config parameters 62 | available. 63 | 64 | Now click **Create Cluster** and go to the **Libraries** tab, where we'll proceed with installing Spline agent. 65 | 66 | ![img_5.png](img_5.png) 67 | 68 | If you have a Spline agent JAR file you can upload it, otherwise you can simply use Maven coordinates, so the agent will be downloaded automatically 69 | from the Maven Central repository. 70 | 71 | ![img_6.png](img_6.png) 72 | 73 | Click **Search Packages**, select **Maven Central** and type "_spline agent bundle_" into the query text field. You'll get a list of available Spline 74 | agent bundles compiled for different Spark and Scala version. 75 | 76 | **Important**: Use a Spline agent bundle that matches the Spark and Scala version of the selected Databricks runtime. 77 | 78 | ![img_7.png](img_7.png) 79 | 80 | Then click **Install** button. 81 | 82 | ![img_8.png](img_8.png) 83 | 84 | The cluster is ready to use, so we can create a new Notebook and start writing our test Spark job: 85 | 86 | ![img_9.png](img_9.png) 87 | 88 | We're almost ready to run some Spark jobs. The last step we need to do is to enable linage tracking on the Spark session. 89 | 90 | In Scala: 91 | ```scala 92 | import za.co.absa.spline.harvester.SparkLineageInitializer._ 93 | 94 | spark.enableLineageTracking() 95 | ``` 96 | 97 | In PySpark: 98 | 99 | ```python 100 | sc._jvm.za.co.absa.spline.harvester.SparkLineageInitializer.enableLineageTracking(spark._jsparkSession) 101 | ``` 102 | 103 | This step has to be done once per Spark session. It could also be done via setting the `spark.sql.queryExecutionListeners` Spark property in the Spark 104 | cluster configuration (see https://github.com/AbsaOSS/spline-spark-agent#initialization), but unfortunately it doesn't work on Databricks. When the 105 | Databricks cluster is booting, the Spark session initializes _before_ the necessary agent library is actually installed on the cluster, resulting in 106 | a `ClassNotFoundError` error, and the cluster fails to start. The workaround is to call `enableLineageTracking()` method explicitly. At the time of 107 | calling that method the Spark session is already initialized and all necessary classes are loaded. 108 | 109 | Now, just run some Spark code as usual. 110 | 111 | We'll use the following example that consists of two jobs. First let's create and save two sample files. 112 | 113 | ```scala 114 | case class Student(id: Int, name: String, addrId: Int) 115 | 116 | case class Address(id: Int, address: String) 117 | 118 | Seq( 119 | Student(111, "Amy Smith", 1), 120 | Student(222, "Bob Brown", 2) 121 | ).toDS.write.mode("overwrite").parquet("/students") 122 | 123 | Seq( 124 | Address(1, "123 Park Ave, San Jose"), 125 | Address(2, "456 Taylor St, Cupertino") 126 | ).toDS.write.mode("overwrite").parquet("/addresses") 127 | ``` 128 | 129 | In the next job, let's read those files, join them and write the result into another file using `append` mode: 130 | 131 | ```scala 132 | val students = spark.read.parquet("/students") 133 | val addresses = spark.read.parquet("/addresses") 134 | 135 | students 136 | .join(addresses) 137 | .where(addresses("id") === students("addrId")) 138 | .select("name", "address") 139 | .write.mode("append").parquet("/student_names_with_addresses") 140 | ``` 141 | 142 | **Note**: Spline agent only tracks persistent actions that result in data written to a file, a table or another persistent location. For example, you 143 | will not see lineage of memory-only actions like `.show()` or `.collect()` 144 | 145 | To see the captured metadata, go to the Spline UI page. 146 | 147 | ![img_11.png](img_10.png) 148 | 149 | If everything is done correctly, you should see three execution events that correspond to three writes in our example. To see the lineage overview of 150 | the data produced by a particular execution event, click on the event name. 151 | 152 | ![img_12.png](img_11.png) 153 | 154 | The graph above represents the high-level lineage of the data produced by the current execution event. It basically shows how the data actually flew 155 | between the data sources and what jobs were involved in the process. 156 | 157 | To see the details of a particular job (execution plan) click on the button on the corresponding node. 158 | 159 | ![img_13.png](img_12.png) 160 | 161 | Here you can see what transformations has been applied on the data, the operation details, input/output data types etc. 162 | 163 | --- 164 | 165 | Copyright 2019 ABSA Group Limited 166 | 167 | you may not use this file except in compliance with the License. 168 | You may obtain a copy of the License at 169 | 170 | http://www.apache.org/licenses/LICENSE-2.0 171 | 172 | Unless required by applicable law or agreed to in writing, software 173 | distributed under the License is distributed on an "AS IS" BASIS, 174 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 175 | See the License for the specific language governing permissions and 176 | limitations under the License. 177 | -------------------------------------------------------------------------------- /spline-on-databricks/img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AbsaOSS/spline-getting-started/96e1187e48746711ec2c582a8bf7f607073bc9e1/spline-on-databricks/img.png -------------------------------------------------------------------------------- /spline-on-databricks/img_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AbsaOSS/spline-getting-started/96e1187e48746711ec2c582a8bf7f607073bc9e1/spline-on-databricks/img_1.png -------------------------------------------------------------------------------- /spline-on-databricks/img_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AbsaOSS/spline-getting-started/96e1187e48746711ec2c582a8bf7f607073bc9e1/spline-on-databricks/img_10.png -------------------------------------------------------------------------------- /spline-on-databricks/img_11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AbsaOSS/spline-getting-started/96e1187e48746711ec2c582a8bf7f607073bc9e1/spline-on-databricks/img_11.png -------------------------------------------------------------------------------- /spline-on-databricks/img_12.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AbsaOSS/spline-getting-started/96e1187e48746711ec2c582a8bf7f607073bc9e1/spline-on-databricks/img_12.png -------------------------------------------------------------------------------- /spline-on-databricks/img_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AbsaOSS/spline-getting-started/96e1187e48746711ec2c582a8bf7f607073bc9e1/spline-on-databricks/img_2.png -------------------------------------------------------------------------------- /spline-on-databricks/img_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AbsaOSS/spline-getting-started/96e1187e48746711ec2c582a8bf7f607073bc9e1/spline-on-databricks/img_3.png -------------------------------------------------------------------------------- /spline-on-databricks/img_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AbsaOSS/spline-getting-started/96e1187e48746711ec2c582a8bf7f607073bc9e1/spline-on-databricks/img_4.png -------------------------------------------------------------------------------- /spline-on-databricks/img_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AbsaOSS/spline-getting-started/96e1187e48746711ec2c582a8bf7f607073bc9e1/spline-on-databricks/img_5.png -------------------------------------------------------------------------------- /spline-on-databricks/img_6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AbsaOSS/spline-getting-started/96e1187e48746711ec2c582a8bf7f607073bc9e1/spline-on-databricks/img_6.png -------------------------------------------------------------------------------- /spline-on-databricks/img_7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AbsaOSS/spline-getting-started/96e1187e48746711ec2c582a8bf7f607073bc9e1/spline-on-databricks/img_7.png -------------------------------------------------------------------------------- /spline-on-databricks/img_8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AbsaOSS/spline-getting-started/96e1187e48746711ec2c582a8bf7f607073bc9e1/spline-on-databricks/img_8.png -------------------------------------------------------------------------------- /spline-on-databricks/img_9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AbsaOSS/spline-getting-started/96e1187e48746711ec2c582a8bf7f607073bc9e1/spline-on-databricks/img_9.png -------------------------------------------------------------------------------- /spline-testing/README.md: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # Testing Spline 4 | 5 | ## Docker-based testing: 6 | Based on `docker-compose` using internally Jmeter, one can run: 7 | 8 | ```bash 9 | ./run_jmetered_spline.py 10 | ``` 11 | 12 | Tests are defined in `./tests` directory, results are appended to the results (created if empty) in `./results` directory next to it. 13 | 14 | There are certain requirements that must be met for the script to be used. Python-modules-wise, the requirements can be installed from the attached `requirements.txt` file using 15 | ```bash 16 | pip install -r requirements.txt 17 | ``` 18 | As for environment requirements, following tools must be available on path: 19 | - `docker` 20 | - `docker-compose` 21 | - `git` 22 | - `mvn` 23 | 24 | --- 25 | 26 | Copyright 2019 ABSA Group Limited 27 | 28 | you may not use this file except in compliance with the License. 29 | You may obtain a copy of the License at 30 | 31 | http://www.apache.org/licenses/LICENSE-2.0 32 | 33 | Unless required by applicable law or agreed to in writing, software 34 | distributed under the License is distributed on an "AS IS" BASIS, 35 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 36 | See the License for the specific language governing permissions and 37 | limitations under the License. 38 | 39 | -------------------------------------------------------------------------------- /spline-testing/jmetered-spline-testing/.env: -------------------------------------------------------------------------------- 1 | # component versions 2 | 3 | ARANGO_DB_VERSION=3.9.2 4 | SPLINE_CORE_VERSION=1.0.0-SNAPSHOT 5 | JMETER_VERSION=5.4 6 | SPLINE_UI_VERSION=0.7.4 7 | 8 | # exposed ports 9 | 10 | ARANGO_DB_PORT=8530 11 | SPLINE_REST_PORT=8082 12 | -------------------------------------------------------------------------------- /spline-testing/jmetered-spline-testing/docker-compose.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '3.8' 3 | services: 4 | arangodb: 5 | image: arangodb:${ARANGO_DB_VERSION} 6 | restart: on-failure 7 | ports: 8 | - ${ARANGO_DB_PORT}:8529 9 | network_mode: "bridge" 10 | environment: 11 | ARANGO_NO_AUTH: 1 12 | 13 | spline: 14 | build: 15 | context: ./spline/rest-gateway 16 | args: 17 | PROJECT_BUILD_FINAL_NAME: spline-rest-server-${SPLINE_CORE_VERSION} 18 | image: local-only/spline-testing/testing-spline-rest-server:${SPLINE_CORE_VERSION} # locally built and used 19 | restart: on-failure 20 | ports: 21 | - ${SPLINE_REST_PORT}:8080 22 | network_mode: "bridge" 23 | command: > 24 | bash -c " 25 | catalina.sh run 26 | " 27 | healthcheck: 28 | # avoid using curl (not natively part of the image); for technical explanation, see 29 | # https://medium.com/bash-tips-and-tricks/part01-tcp-udp-request-with-a-native-bash-feature-and-without-curl-wget-9dcef59c30aa 30 | test: timeout 10s bash -c ':> /dev/tcp/172.17.0.1/${SPLINE_REST_PORT}' 31 | interval: 10s 32 | timeout: 20s 33 | retries: 20 34 | environment: 35 | SPLINE_DATABASE_CONNECTION_URL: 'arangodb://172.17.0.1:${ARANGO_DB_PORT}/spline' 36 | # by default /dev/random is used which may block 37 | CATALINA_OPTS: "-Dsecurerandom.source=file:/dev/./urandom -Djava.security.egd=file:/dev/./urandom" 38 | links: 39 | - arangodb 40 | depends_on: 41 | spline-init-db: 42 | condition: service_healthy 43 | 44 | spline-init-db: 45 | build: 46 | context: ./spline/admin 47 | args: 48 | PROJECT_BUILD_FINAL_NAME: admin-${SPLINE_CORE_VERSION} 49 | IMAGE_NAME: spline-admin 50 | image: local-only/spline-testing/testing-spline-db-admin:${SPLINE_CORE_VERSION} # locally built and used 51 | restart: on-failure 52 | network_mode: "bridge" 53 | entrypoint: > 54 | tini -g -- bash -c " 55 | until curl --output /dev/null --silent --get --fail http://172.17.0.1:${ARANGO_DB_PORT}/_admin/server/availability 56 | do 57 | echo waiting for ArangoDB server to be ready to run spline-admin:${SPLINE_CORE_VERSION} to init the spline DB 58 | sleep 5 59 | done 60 | bash ./entrypoint.sh db-init arangodb://172.17.0.1:${ARANGO_DB_PORT}/spline -s && sleep infinity 61 | " 62 | # ^ Waiting for jmeter image to finish first with measurements, teardown only after that 63 | healthcheck: 64 | # health ok = spline DB exists (init successfully done) 65 | test: curl --output /dev/null --silent --get --fail http://172.17.0.1:${ARANGO_DB_PORT}/_db/spline/_admin/server/availability 66 | interval: 10s 67 | timeout: 20s 68 | retries: 20 69 | links: 70 | - arangodb 71 | 72 | jmeter: 73 | image: justb4/jmeter:${JMETER_VERSION} 74 | network_mode: "bridge" 75 | entrypoint: busybox 76 | # escaping like $${item} is needed, just '$item' would be replaced by docker-compose. https://github.com/docker/compose/issues/4189 77 | # -J"user.properties=/var/jmeter/user.properties" with sample_variables being set does not seem to work, so it is set explicitly below 78 | command: > 79 | sh -c ' 80 | echo "Purging any previous result if present in 'results' dir" 81 | rm -rf /var/jmeter/results/* 82 | for item in `ls /var/jmeter/tests`; do 83 | echo "Running jmeter measuring for $${item}" 84 | jmeter -n -t /var/jmeter/lineage-multi.jmx -l /var/jmeter/results/$${item}.res -Jsample_variables=graphType,operationCount,attributeCount,readCount -J"config_file=/var/jmeter/tests/$${item}" -Jspline_base=http://172.17.0.1:${SPLINE_REST_PORT} 85 | done 86 | ' 87 | volumes: 88 | - .:/var/jmeter 89 | links: 90 | - spline 91 | depends_on: 92 | spline: 93 | condition: service_healthy 94 | -------------------------------------------------------------------------------- /spline-testing/jmetered-spline-testing/lineage-multi.jmx: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | false 7 | true 8 | false 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | spline_base 19 | ${__P(spline_base)} 20 | = 21 | 22 | 23 | config_file 24 | ${__P(config_file)} 25 | = 26 | 27 | 28 | contentType 29 | application/vnd.absa.spline.producer.v1.1+json 30 | = 31 | 32 | 33 | method 34 | POST 35 | = 36 | 37 | 38 | 39 | 40 | 41 | false 42 | 43 | saveConfig 44 | 45 | 46 | true 47 | true 48 | true 49 | 50 | true 51 | true 52 | true 53 | true 54 | false 55 | true 56 | true 57 | false 58 | false 59 | false 60 | true 61 | false 62 | false 63 | false 64 | true 65 | 0 66 | true 67 | true 68 | true 69 | true 70 | true 71 | true 72 | true 73 | true 74 | 75 | 76 | 77 | 78 | 79 | 80 | continue 81 | 82 | false 83 | 1 84 | 85 | 1 86 | 1 87 | false 88 | 89 | 90 | true 91 | 92 | 93 | 94 | {config_line} 95 | 96 | 97 | 98 | # 99 | 100 | ${config_file} 101 | false 102 | false 103 | false 104 | shareMode.all 105 | true 106 | config_line 107 | 108 | 109 | 110 | graphType; operationCount; attributeCount; readCount 111 | extraInfo.graph-type; extraInfo.operationCount; extraInfo.attributeCount; extraInfo.readCount 112 | 1 113 | variable 114 | config_line 115 | ${graphType}; ${operationCount}; ${attributeCount}; ${readCount} 116 | 117 | 118 | 119 | graphType 120 | extraInfo.graph-type 121 | 1 122 | variable 123 | config_line 124 | "" 125 | 126 | 127 | 128 | ${__groovy(vars.get("config_line").substring(0\,1) == "{" )} 129 | false 130 | true 131 | 132 | 133 | 134 | true 135 | 136 | 137 | 138 | false 139 | ${config_line} 140 | = 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | ${spline_base}/producer/execution-plans 149 | ${method} 150 | true 151 | false 152 | true 153 | false 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | User-Agent 163 | ApacheJMeter 164 | 165 | 166 | Content-Type 167 | ${contentType} 168 | 169 | 170 | X-absa-graphType 171 | ${graphType} 172 | 173 | 174 | X-absa-readCount 175 | ${readCount} 176 | 177 | 178 | X-absa-operationCount 179 | ${operationCount} 180 | 181 | 182 | X-absa-attributeCount 183 | ${attributeCount} 184 | 185 | 186 | 187 | 188 | 189 | 190 | false 191 | true 192 | false 193 | 194 | 195 | 196 | true 197 | 198 | 199 | log.info("**** graphType = " + vars.get("graphType")) 200 | log.info("**** operationCount = " + vars.get("operationCount")) 201 | log.info("**** attributeCount = " + vars.get("attributeCount")) 202 | log.info("**** readCount = " + vars.get("readCount")) 203 | 204 | log.info("**** sample_variables = " + vars.get("sample_variables")) 205 | log.info("**** user.properties = " + vars.get("user.properties")) 206 | groovy 207 | 208 | 209 | 210 | 211 | ${__groovy(vars.get("config_line").substring(0\,1) == "[" )} 212 | false 213 | true 214 | 215 | 216 | 217 | true 218 | 219 | 220 | 221 | false 222 | ${config_line} 223 | = 224 | 225 | 226 | 227 | 228 | 229 | 230 | 231 | ${spline_base}/producer/execution-events 232 | ${method} 233 | true 234 | false 235 | true 236 | false 237 | 238 | 239 | 240 | 241 | 242 | 243 | 244 | 245 | User-Agent 246 | ApacheJMeter 247 | 248 | 249 | Content-Type 250 | ${contentType} 251 | 252 | 253 | 254 | 255 | 256 | 257 | false 258 | true 259 | false 260 | 261 | 262 | 263 | true 264 | 265 | 266 | log.info("**** graphType = " + vars.get("graphType")) 267 | log.info("**** operationCount = " + vars.get("operationCount")) 268 | log.info("**** attributeCount = " + vars.get("attributeCount")) 269 | log.info("**** readCount = " + vars.get("readCount")) 270 | 271 | log.info("**** sample_variables = " + vars.get("sample_variables")) 272 | log.info("**** user.properties = " + vars.get("user.properties")) 273 | groovy 274 | 275 | 276 | 277 | 278 | 279 | 280 | 281 | 282 | -------------------------------------------------------------------------------- /spline-testing/jmetered-spline-testing/reference/chain-lineage-1-2001@50reads-3ops-4attr.json.txt.res: -------------------------------------------------------------------------------- 1 | timeStamp,elapsed,label,responseCode,responseMessage,threadName,dataType,success,failureMessage,bytes,sentBytes,grpThreads,allThreads,URL,Latency,IdleTime,Connect,"graphType","operationCount","attributeCount","readCount" 2 | 1663859496422,2001,posting lineage plan,201,,simple post group 1-1,text,true,,300,9107,1,1,http://172.17.0.1:8080/producer/execution-plans,2000,0,31,Chain,3,4,1 3 | 1663859498546,71,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,71,0,0,Chain, 3, 4, 1 4 | 1663859498620,47,posting lineage plan,201,,simple post group 1-1,text,true,,300,12454,1,1,http://172.17.0.1:8080/producer/execution-plans,47,0,0,Chain,3,4,51 5 | 1663859498671,9,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,9,0,0,Chain, 3, 4, 51 6 | 1663859498684,50,posting lineage plan,201,,simple post group 1-1,text,true,,300,15858,1,1,http://172.17.0.1:8080/producer/execution-plans,50,0,0,Chain,3,4,101 7 | 1663859498737,9,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,9,0,0,Chain, 3, 4, 101 8 | 1663859498749,54,posting lineage plan,201,,simple post group 1-1,text,true,,300,19309,1,1,http://172.17.0.1:8080/producer/execution-plans,54,0,0,Chain,3,4,151 9 | 1663859498806,9,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,9,0,0,Chain, 3, 4, 151 10 | 1663859498820,56,posting lineage plan,201,,simple post group 1-1,text,true,,300,22759,1,1,http://172.17.0.1:8080/producer/execution-plans,56,0,0,Chain,3,4,201 11 | 1663859498879,9,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,9,0,0,Chain, 3, 4, 201 12 | 1663859498892,63,posting lineage plan,201,,simple post group 1-1,text,true,,300,26209,1,1,http://172.17.0.1:8080/producer/execution-plans,63,0,0,Chain,3,4,251 13 | 1663859498957,10,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,10,0,0,Chain, 3, 4, 251 14 | 1663859498971,82,posting lineage plan,201,,simple post group 1-1,text,true,,300,29659,1,1,http://172.17.0.1:8080/producer/execution-plans,82,0,0,Chain,3,4,301 15 | 1663859499056,9,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,9,0,0,Chain, 3, 4, 301 16 | 1663859499071,83,posting lineage plan,201,,simple post group 1-1,text,true,,300,33109,1,1,http://172.17.0.1:8080/producer/execution-plans,83,0,0,Chain,3,4,351 17 | 1663859499158,9,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,9,0,0,Chain, 3, 4, 351 18 | 1663859499172,82,posting lineage plan,201,,simple post group 1-1,text,true,,300,36559,1,1,http://172.17.0.1:8080/producer/execution-plans,81,0,0,Chain,3,4,401 19 | 1663859499257,9,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,9,0,0,Chain, 3, 4, 401 20 | 1663859499270,120,posting lineage plan,201,,simple post group 1-1,text,true,,300,40009,1,1,http://172.17.0.1:8080/producer/execution-plans,120,0,0,Chain,3,4,451 21 | 1663859499392,10,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,10,0,0,Chain, 3, 4, 451 22 | 1663859499405,97,posting lineage plan,201,,simple post group 1-1,text,true,,300,43459,1,1,http://172.17.0.1:8080/producer/execution-plans,97,0,0,Chain,3,4,501 23 | 1663859499505,8,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,8,0,0,Chain, 3, 4, 501 24 | 1663859499517,157,posting lineage plan,201,,simple post group 1-1,text,true,,300,46909,1,1,http://172.17.0.1:8080/producer/execution-plans,157,0,0,Chain,3,4,551 25 | 1663859499677,9,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,9,0,0,Chain, 3, 4, 551 26 | 1663859499690,111,posting lineage plan,201,,simple post group 1-1,text,true,,300,50359,1,1,http://172.17.0.1:8080/producer/execution-plans,111,0,0,Chain,3,4,601 27 | 1663859499804,8,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,8,0,0,Chain, 3, 4, 601 28 | 1663859499816,120,posting lineage plan,201,,simple post group 1-1,text,true,,300,53809,1,1,http://172.17.0.1:8080/producer/execution-plans,120,0,0,Chain,3,4,651 29 | 1663859499938,8,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,8,0,0,Chain, 3, 4, 651 30 | 1663859499950,135,posting lineage plan,201,,simple post group 1-1,text,true,,300,57259,1,1,http://172.17.0.1:8080/producer/execution-plans,134,0,0,Chain,3,4,701 31 | 1663859500087,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Chain, 3, 4, 701 32 | 1663859500099,152,posting lineage plan,201,,simple post group 1-1,text,true,,300,60709,1,1,http://172.17.0.1:8080/producer/execution-plans,152,0,0,Chain,3,4,751 33 | 1663859500253,9,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,9,0,0,Chain, 3, 4, 751 34 | 1663859500267,165,posting lineage plan,201,,simple post group 1-1,text,true,,300,64159,1,1,http://172.17.0.1:8080/producer/execution-plans,165,0,0,Chain,3,4,801 35 | 1663859500434,8,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,8,0,0,Chain, 3, 4, 801 36 | 1663859500446,153,posting lineage plan,201,,simple post group 1-1,text,true,,300,67609,1,1,http://172.17.0.1:8080/producer/execution-plans,153,0,0,Chain,3,4,851 37 | 1663859500602,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Chain, 3, 4, 851 38 | 1663859500614,168,posting lineage plan,201,,simple post group 1-1,text,true,,300,71059,1,1,http://172.17.0.1:8080/producer/execution-plans,168,0,0,Chain,3,4,901 39 | 1663859500784,8,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,8,0,0,Chain, 3, 4, 901 40 | 1663859500797,201,posting lineage plan,201,,simple post group 1-1,text,true,,300,74509,1,1,http://172.17.0.1:8080/producer/execution-plans,201,0,0,Chain,3,4,951 41 | 1663859501000,11,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,11,0,0,Chain, 3, 4, 951 42 | 1663859501015,187,posting lineage plan,201,,simple post group 1-1,text,true,,300,77962,1,1,http://172.17.0.1:8080/producer/execution-plans,187,0,0,Chain,3,4,1001 43 | 1663859501204,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Chain, 3, 4, 1001 44 | 1663859501216,190,posting lineage plan,201,,simple post group 1-1,text,true,,300,81463,1,1,http://172.17.0.1:8080/producer/execution-plans,189,0,0,Chain,3,4,1051 45 | 1663859501408,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Chain, 3, 4, 1051 46 | 1663859501421,196,posting lineage plan,201,,simple post group 1-1,text,true,,300,84963,1,1,http://172.17.0.1:8080/producer/execution-plans,196,0,0,Chain,3,4,1101 47 | 1663859501619,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Chain, 3, 4, 1101 48 | 1663859501631,210,posting lineage plan,201,,simple post group 1-1,text,true,,300,88463,1,1,http://172.17.0.1:8080/producer/execution-plans,210,0,0,Chain,3,4,1151 49 | 1663859501842,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Chain, 3, 4, 1151 50 | 1663859501855,224,posting lineage plan,201,,simple post group 1-1,text,true,,300,91963,1,1,http://172.17.0.1:8080/producer/execution-plans,224,0,0,Chain,3,4,1201 51 | 1663859502081,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Chain, 3, 4, 1201 52 | 1663859502093,252,posting lineage plan,201,,simple post group 1-1,text,true,,300,95463,1,1,http://172.17.0.1:8080/producer/execution-plans,251,0,0,Chain,3,4,1251 53 | 1663859502347,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Chain, 3, 4, 1251 54 | 1663859502360,237,posting lineage plan,201,,simple post group 1-1,text,true,,300,98963,1,1,http://172.17.0.1:8080/producer/execution-plans,237,0,0,Chain,3,4,1301 55 | 1663859502599,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Chain, 3, 4, 1301 56 | 1663859502612,237,posting lineage plan,201,,simple post group 1-1,text,true,,300,102464,1,1,http://172.17.0.1:8080/producer/execution-plans,237,0,0,Chain,3,4,1351 57 | 1663859502851,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Chain, 3, 4, 1351 58 | 1663859502865,252,posting lineage plan,201,,simple post group 1-1,text,true,,300,105964,1,1,http://172.17.0.1:8080/producer/execution-plans,251,0,0,Chain,3,4,1401 59 | 1663859503119,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Chain, 3, 4, 1401 60 | 1663859503131,296,posting lineage plan,201,,simple post group 1-1,text,true,,300,109464,1,1,http://172.17.0.1:8080/producer/execution-plans,296,0,0,Chain,3,4,1451 61 | 1663859503429,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Chain, 3, 4, 1451 62 | 1663859503443,286,posting lineage plan,201,,simple post group 1-1,text,true,,300,112964,1,1,http://172.17.0.1:8080/producer/execution-plans,286,0,0,Chain,3,4,1501 63 | 1663859503732,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Chain, 3, 4, 1501 64 | 1663859503745,275,posting lineage plan,201,,simple post group 1-1,text,true,,300,116464,1,1,http://172.17.0.1:8080/producer/execution-plans,275,0,0,Chain,3,4,1551 65 | 1663859504024,8,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,8,0,0,Chain, 3, 4, 1551 66 | 1663859504040,276,posting lineage plan,201,,simple post group 1-1,text,true,,300,119964,1,1,http://172.17.0.1:8080/producer/execution-plans,276,0,0,Chain,3,4,1601 67 | 1663859504319,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Chain, 3, 4, 1601 68 | 1663859504332,296,posting lineage plan,201,,simple post group 1-1,text,true,,300,123464,1,1,http://172.17.0.1:8080/producer/execution-plans,296,0,0,Chain,3,4,1651 69 | 1663859504630,8,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,8,0,0,Chain, 3, 4, 1651 70 | 1663859504645,324,posting lineage plan,201,,simple post group 1-1,text,true,,300,126964,1,1,http://172.17.0.1:8080/producer/execution-plans,324,0,0,Chain,3,4,1701 71 | 1663859504971,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Chain, 3, 4, 1701 72 | 1663859504985,322,posting lineage plan,201,,simple post group 1-1,text,true,,300,130464,1,1,http://172.17.0.1:8080/producer/execution-plans,322,0,0,Chain,3,4,1751 73 | 1663859505310,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Chain, 3, 4, 1751 74 | 1663859505324,319,posting lineage plan,201,,simple post group 1-1,text,true,,300,133964,1,1,http://172.17.0.1:8080/producer/execution-plans,318,0,0,Chain,3,4,1801 75 | 1663859505645,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Chain, 3, 4, 1801 76 | 1663859505659,319,posting lineage plan,201,,simple post group 1-1,text,true,,300,137464,1,1,http://172.17.0.1:8080/producer/execution-plans,319,0,0,Chain,3,4,1851 77 | 1663859505981,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Chain, 3, 4, 1851 78 | 1663859505995,351,posting lineage plan,201,,simple post group 1-1,text,true,,300,140964,1,1,http://172.17.0.1:8080/producer/execution-plans,351,0,0,Chain,3,4,1901 79 | 1663859506349,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Chain, 3, 4, 1901 80 | 1663859506364,349,posting lineage plan,201,,simple post group 1-1,text,true,,300,144464,1,1,http://172.17.0.1:8080/producer/execution-plans,349,0,0,Chain,3,4,1951 81 | 1663859506716,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Chain, 3, 4, 1951 82 | 1663859506739,363,posting lineage plan,201,,simple post group 1-1,text,true,,300,147964,1,1,http://172.17.0.1:8080/producer/execution-plans,362,0,0,Chain,3,4,2001 83 | 1663859507104,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Chain, 3, 4, 2001 84 | -------------------------------------------------------------------------------- /spline-testing/jmetered-spline-testing/reference/chain-lineage-5reads-1-501@50ops-4attr.json.txt.res: -------------------------------------------------------------------------------- 1 | timeStamp,elapsed,label,responseCode,responseMessage,threadName,dataType,success,failureMessage,bytes,sentBytes,grpThreads,allThreads,URL,Latency,IdleTime,Connect,"graphType","operationCount","attributeCount","readCount" 2 | 1663859510016,56,posting lineage plan,201,,simple post group 1-1,text,true,,300,4665,1,1,http://172.17.0.1:8080/producer/execution-plans,55,0,33,Chain,1,4,5 3 | 1663859510204,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Chain, 1, 4, 5 4 | 1663859510220,41,posting lineage plan,201,,simple post group 1-1,text,true,,300,122371,1,1,http://172.17.0.1:8080/producer/execution-plans,41,0,0,Chain,51,4,5 5 | 1663859510271,8,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,8,0,0,Chain, 51, 4, 5 6 | 1663859510293,63,posting lineage plan,201,,simple post group 1-1,text,true,,300,240123,1,1,http://172.17.0.1:8080/producer/execution-plans,63,0,0,Chain,101,4,5 7 | 1663859510366,8,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,8,0,0,Chain, 101, 4, 5 8 | 1663859510387,66,posting lineage plan,201,,simple post group 1-1,text,true,,300,357874,1,1,http://172.17.0.1:8080/producer/execution-plans,65,0,0,Chain,151,4,5 9 | 1663859510461,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Chain, 151, 4, 5 10 | 1663859510485,65,posting lineage plan,201,,simple post group 1-1,text,true,,300,475624,1,1,http://172.17.0.1:8080/producer/execution-plans,64,0,0,Chain,201,4,5 11 | 1663859510562,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Chain, 201, 4, 5 12 | 1663859510585,61,posting lineage plan,201,,simple post group 1-1,text,true,,300,593374,1,1,http://172.17.0.1:8080/producer/execution-plans,61,0,0,Chain,251,4,5 13 | 1663859510665,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Chain, 251, 4, 5 14 | 1663859510696,57,posting lineage plan,201,,simple post group 1-1,text,true,,300,711124,1,1,http://172.17.0.1:8080/producer/execution-plans,57,0,0,Chain,301,4,5 15 | 1663859510765,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Chain, 301, 4, 5 16 | 1663859510796,77,posting lineage plan,201,,simple post group 1-1,text,true,,300,828874,1,1,http://172.17.0.1:8080/producer/execution-plans,77,0,0,Chain,351,4,5 17 | 1663859510895,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Chain, 351, 4, 5 18 | 1663859510934,86,posting lineage plan,201,,simple post group 1-1,text,true,,300,946624,1,1,http://172.17.0.1:8080/producer/execution-plans,86,0,0,Chain,401,4,5 19 | 1663859511037,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Chain, 401, 4, 5 20 | 1663859511080,78,posting lineage plan,201,,simple post group 1-1,text,true,,300,1064375,1,1,http://172.17.0.1:8080/producer/execution-plans,77,0,0,Chain,451,4,5 21 | 1663859511186,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Chain, 451, 4, 5 22 | 1663859511227,82,posting lineage plan,201,,simple post group 1-1,text,true,,300,1182125,1,1,http://172.17.0.1:8080/producer/execution-plans,82,0,0,Chain,501,4,5 23 | 1663859511334,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Chain, 501, 4, 5 24 | -------------------------------------------------------------------------------- /spline-testing/jmetered-spline-testing/reference/chain-lineage-5reads-3ops-1-501@50attr.json.txt.res: -------------------------------------------------------------------------------- 1 | timeStamp,elapsed,label,responseCode,responseMessage,threadName,dataType,success,failureMessage,bytes,sentBytes,grpThreads,allThreads,URL,Latency,IdleTime,Connect,"graphType","operationCount","attributeCount","readCount" 2 | 1663859514319,52,posting lineage plan,201,,simple post group 1-1,text,true,,300,4017,1,1,http://172.17.0.1:8080/producer/execution-plans,52,0,34,Chain,3,1,5 3 | 1663859514515,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Chain, 3, 1, 5 4 | 1663859514529,21,posting lineage plan,201,,simple post group 1-1,text,true,,300,93314,1,1,http://172.17.0.1:8080/producer/execution-plans,21,0,0,Chain,3,51,5 5 | 1663859514560,8,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,8,0,0,Chain, 3, 51, 5 6 | 1663859514583,23,posting lineage plan,201,,simple post group 1-1,text,true,,300,182669,1,1,http://172.17.0.1:8080/producer/execution-plans,23,0,0,Chain,3,101,5 7 | 1663859514614,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Chain, 3, 101, 5 8 | 1663859514631,26,posting lineage plan,201,,simple post group 1-1,text,true,,300,272070,1,1,http://172.17.0.1:8080/producer/execution-plans,26,0,0,Chain,3,151,5 9 | 1663859514665,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Chain, 3, 151, 5 10 | 1663859514683,29,posting lineage plan,201,,simple post group 1-1,text,true,,300,361470,1,1,http://172.17.0.1:8080/producer/execution-plans,29,0,0,Chain,3,201,5 11 | 1663859514720,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Chain, 3, 201, 5 12 | 1663859514740,37,posting lineage plan,201,,simple post group 1-1,text,true,,300,450870,1,1,http://172.17.0.1:8080/producer/execution-plans,37,0,0,Chain,3,251,5 13 | 1663859514786,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Chain, 3, 251, 5 14 | 1663859514810,53,posting lineage plan,201,,simple post group 1-1,text,true,,300,540270,1,1,http://172.17.0.1:8080/producer/execution-plans,53,0,0,Chain,3,301,5 15 | 1663859514880,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Chain, 3, 301, 5 16 | 1663859514907,49,posting lineage plan,201,,simple post group 1-1,text,true,,300,629670,1,1,http://172.17.0.1:8080/producer/execution-plans,49,0,0,Chain,3,351,5 17 | 1663859514967,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Chain, 3, 351, 5 18 | 1663859514996,46,posting lineage plan,201,,simple post group 1-1,text,true,,300,719070,1,1,http://172.17.0.1:8080/producer/execution-plans,46,0,0,Chain,3,401,5 19 | 1663859515053,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Chain, 3, 401, 5 20 | 1663859515085,53,posting lineage plan,201,,simple post group 1-1,text,true,,300,808470,1,1,http://172.17.0.1:8080/producer/execution-plans,53,0,0,Chain,3,451,5 21 | 1663859515151,9,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,8,0,0,Chain, 3, 451, 5 22 | 1663859515186,59,posting lineage plan,201,,simple post group 1-1,text,true,,300,897870,1,1,http://172.17.0.1:8080/producer/execution-plans,59,0,0,Chain,3,501,5 23 | 1663859515262,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Chain, 3, 501, 5 24 | -------------------------------------------------------------------------------- /spline-testing/jmetered-spline-testing/reference/diamond-lineage-1-1001@50reads-3ops-4attr.json.txt.res: -------------------------------------------------------------------------------- 1 | timeStamp,elapsed,label,responseCode,responseMessage,threadName,dataType,success,failureMessage,bytes,sentBytes,grpThreads,allThreads,URL,Latency,IdleTime,Connect,"graphType","operationCount","attributeCount","readCount" 2 | 1663859518330,53,posting lineage plan,201,,simple post group 1-1,text,true,,300,9499,1,1,http://172.17.0.1:8080/producer/execution-plans,52,0,34,Diamond,3,4,1 3 | 1663859518537,8,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,8,0,0,Diamond, 3, 4, 1 4 | 1663859518549,19,posting lineage plan,201,,simple post group 1-1,text,true,,300,12848,1,1,http://172.17.0.1:8080/producer/execution-plans,19,0,0,Diamond,3,4,51 5 | 1663859518572,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Diamond, 3, 4, 51 6 | 1663859518583,27,posting lineage plan,201,,simple post group 1-1,text,true,,300,16252,1,1,http://172.17.0.1:8080/producer/execution-plans,27,0,0,Diamond,3,4,101 7 | 1663859518613,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Diamond, 3, 4, 101 8 | 1663859518623,42,posting lineage plan,201,,simple post group 1-1,text,true,,300,19703,1,1,http://172.17.0.1:8080/producer/execution-plans,41,0,0,Diamond,3,4,151 9 | 1663859518668,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Diamond, 3, 4, 151 10 | 1663859518678,49,posting lineage plan,201,,simple post group 1-1,text,true,,300,23153,1,1,http://172.17.0.1:8080/producer/execution-plans,49,0,0,Diamond,3,4,201 11 | 1663859518730,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Diamond, 3, 4, 201 12 | 1663859518742,53,posting lineage plan,201,,simple post group 1-1,text,true,,300,26603,1,1,http://172.17.0.1:8080/producer/execution-plans,53,0,0,Diamond,3,4,251 13 | 1663859518797,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Diamond, 3, 4, 251 14 | 1663859518807,71,posting lineage plan,201,,simple post group 1-1,text,true,,300,30053,1,1,http://172.17.0.1:8080/producer/execution-plans,71,0,0,Diamond,3,4,301 15 | 1663859518881,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Diamond, 3, 4, 301 16 | 1663859518892,74,posting lineage plan,201,,simple post group 1-1,text,true,,300,33503,1,1,http://172.17.0.1:8080/producer/execution-plans,74,0,0,Diamond,3,4,351 17 | 1663859518969,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Diamond, 3, 4, 351 18 | 1663859518978,80,posting lineage plan,201,,simple post group 1-1,text,true,,300,36953,1,1,http://172.17.0.1:8080/producer/execution-plans,79,0,0,Diamond,3,4,401 19 | 1663859519060,5,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,5,0,0,Diamond, 3, 4, 401 20 | 1663859519070,81,posting lineage plan,201,,simple post group 1-1,text,true,,300,40403,1,1,http://172.17.0.1:8080/producer/execution-plans,81,0,0,Diamond,3,4,451 21 | 1663859519153,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Diamond, 3, 4, 451 22 | 1663859519163,94,posting lineage plan,201,,simple post group 1-1,text,true,,300,43853,1,1,http://172.17.0.1:8080/producer/execution-plans,94,0,0,Diamond,3,4,501 23 | 1663859519259,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Diamond, 3, 4, 501 24 | 1663859519269,94,posting lineage plan,201,,simple post group 1-1,text,true,,300,47303,1,1,http://172.17.0.1:8080/producer/execution-plans,93,0,0,Diamond,3,4,551 25 | 1663859519364,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Diamond, 3, 4, 551 26 | 1663859519374,100,posting lineage plan,201,,simple post group 1-1,text,true,,300,50753,1,1,http://172.17.0.1:8080/producer/execution-plans,100,0,0,Diamond,3,4,601 27 | 1663859519476,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Diamond, 3, 4, 601 28 | 1663859519487,113,posting lineage plan,201,,simple post group 1-1,text,true,,300,54203,1,1,http://172.17.0.1:8080/producer/execution-plans,113,0,0,Diamond,3,4,651 29 | 1663859519602,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Diamond, 3, 4, 651 30 | 1663859519612,124,posting lineage plan,201,,simple post group 1-1,text,true,,300,57653,1,1,http://172.17.0.1:8080/producer/execution-plans,124,0,0,Diamond,3,4,701 31 | 1663859519738,5,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,5,0,0,Diamond, 3, 4, 701 32 | 1663859519748,136,posting lineage plan,201,,simple post group 1-1,text,true,,300,61103,1,1,http://172.17.0.1:8080/producer/execution-plans,136,0,0,Diamond,3,4,751 33 | 1663859519886,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Diamond, 3, 4, 751 34 | 1663859519897,146,posting lineage plan,201,,simple post group 1-1,text,true,,300,64553,1,1,http://172.17.0.1:8080/producer/execution-plans,146,0,0,Diamond,3,4,801 35 | 1663859520046,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Diamond, 3, 4, 801 36 | 1663859520057,168,posting lineage plan,201,,simple post group 1-1,text,true,,300,68003,1,1,http://172.17.0.1:8080/producer/execution-plans,167,0,0,Diamond,3,4,851 37 | 1663859520226,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Diamond, 3, 4, 851 38 | 1663859520237,159,posting lineage plan,201,,simple post group 1-1,text,true,,300,71453,1,1,http://172.17.0.1:8080/producer/execution-plans,159,0,0,Diamond,3,4,901 39 | 1663859520398,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Diamond, 3, 4, 901 40 | 1663859520408,171,posting lineage plan,201,,simple post group 1-1,text,true,,300,74903,1,1,http://172.17.0.1:8080/producer/execution-plans,171,0,0,Diamond,3,4,951 41 | 1663859520581,5,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,5,0,0,Diamond, 3, 4, 951 42 | 1663859520592,168,posting lineage plan,201,,simple post group 1-1,text,true,,300,78356,1,1,http://172.17.0.1:8080/producer/execution-plans,168,0,0,Diamond,3,4,1001 43 | 1663859520762,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Diamond, 3, 4, 1001 44 | -------------------------------------------------------------------------------- /spline-testing/jmetered-spline-testing/reference/diamond-lineage-5reads-1-501@50ops-4attr.json.txt.res: -------------------------------------------------------------------------------- 1 | timeStamp,elapsed,label,responseCode,responseMessage,threadName,dataType,success,failureMessage,bytes,sentBytes,grpThreads,allThreads,URL,Latency,IdleTime,Connect,"graphType","operationCount","attributeCount","readCount" 2 | 1663859523673,58,posting lineage plan,201,,simple post group 1-1,text,true,,300,4667,1,1,http://172.17.0.1:8080/producer/execution-plans,57,0,30,Diamond,1,4,5 3 | 1663859523876,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Diamond, 1, 4, 5 4 | 1663859523892,19,posting lineage plan,201,,simple post group 1-1,text,true,,300,132125,1,1,http://172.17.0.1:8080/producer/execution-plans,19,0,0,Diamond,51,4,5 5 | 1663859523920,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Diamond, 51, 4, 5 6 | 1663859523937,20,posting lineage plan,201,,simple post group 1-1,text,true,,300,259627,1,1,http://172.17.0.1:8080/producer/execution-plans,20,0,0,Diamond,101,4,5 7 | 1663859523963,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Diamond, 101, 4, 5 8 | 1663859523981,28,posting lineage plan,201,,simple post group 1-1,text,true,,300,387128,1,1,http://172.17.0.1:8080/producer/execution-plans,28,0,0,Diamond,151,4,5 9 | 1663859524017,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Diamond, 151, 4, 5 10 | 1663859524042,35,posting lineage plan,201,,simple post group 1-1,text,true,,300,514628,1,1,http://172.17.0.1:8080/producer/execution-plans,35,0,0,Diamond,201,4,5 11 | 1663859524086,5,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,5,0,0,Diamond, 201, 4, 5 12 | 1663859524114,44,posting lineage plan,201,,simple post group 1-1,text,true,,300,642128,1,1,http://172.17.0.1:8080/producer/execution-plans,44,0,0,Diamond,251,4,5 13 | 1663859524177,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Diamond, 251, 4, 5 14 | 1663859524206,51,posting lineage plan,201,,simple post group 1-1,text,true,,300,769628,1,1,http://172.17.0.1:8080/producer/execution-plans,51,0,0,Diamond,301,4,5 15 | 1663859524269,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Diamond, 301, 4, 5 16 | 1663859524302,56,posting lineage plan,201,,simple post group 1-1,text,true,,300,897128,1,1,http://172.17.0.1:8080/producer/execution-plans,56,0,0,Diamond,351,4,5 17 | 1663859524372,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Diamond, 351, 4, 5 18 | 1663859524416,63,posting lineage plan,201,,simple post group 1-1,text,true,,300,1024629,1,1,http://172.17.0.1:8080/producer/execution-plans,63,0,0,Diamond,401,4,5 19 | 1663859524494,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Diamond, 401, 4, 5 20 | 1663859524530,81,posting lineage plan,201,,simple post group 1-1,text,true,,300,1152129,1,1,http://172.17.0.1:8080/producer/execution-plans,81,0,0,Diamond,451,4,5 21 | 1663859524635,5,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,5,0,0,Diamond, 451, 4, 5 22 | 1663859524678,80,posting lineage plan,201,,simple post group 1-1,text,true,,300,1279629,1,1,http://172.17.0.1:8080/producer/execution-plans,80,0,0,Diamond,501,4,5 23 | 1663859524785,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Diamond, 501, 4, 5 24 | -------------------------------------------------------------------------------- /spline-testing/jmetered-spline-testing/reference/diamond-lineage-5reads-3ops-1-501@50attr.json.txt.res: -------------------------------------------------------------------------------- 1 | timeStamp,elapsed,label,responseCode,responseMessage,threadName,dataType,success,failureMessage,bytes,sentBytes,grpThreads,allThreads,URL,Latency,IdleTime,Connect,"graphType","operationCount","attributeCount","readCount" 2 | 1663859527841,59,posting lineage plan,201,,simple post group 1-1,text,true,,300,4175,1,1,http://172.17.0.1:8080/producer/execution-plans,58,0,37,Diamond,3,1,5 3 | 1663859528038,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Diamond, 3, 1, 5 4 | 1663859528054,20,posting lineage plan,201,,simple post group 1-1,text,true,,300,97374,1,1,http://172.17.0.1:8080/producer/execution-plans,20,0,0,Diamond,3,51,5 5 | 1663859528083,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Diamond, 3, 51, 5 6 | 1663859528099,61,posting lineage plan,201,,simple post group 1-1,text,true,,300,190629,1,1,http://172.17.0.1:8080/producer/execution-plans,61,0,0,Diamond,3,101,5 7 | 1663859528174,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Diamond, 3, 101, 5 8 | 1663859528197,31,posting lineage plan,201,,simple post group 1-1,text,true,,300,283930,1,1,http://172.17.0.1:8080/producer/execution-plans,31,0,0,Diamond,3,151,5 9 | 1663859528235,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Diamond, 3, 151, 5 10 | 1663859528256,33,posting lineage plan,201,,simple post group 1-1,text,true,,300,377230,1,1,http://172.17.0.1:8080/producer/execution-plans,32,0,0,Diamond,3,201,5 11 | 1663859528299,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Diamond, 3, 201, 5 12 | 1663859528322,37,posting lineage plan,201,,simple post group 1-1,text,true,,300,470530,1,1,http://172.17.0.1:8080/producer/execution-plans,37,0,0,Diamond,3,251,5 13 | 1663859528368,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Diamond, 3, 251, 5 14 | 1663859528395,40,posting lineage plan,201,,simple post group 1-1,text,true,,300,563830,1,1,http://172.17.0.1:8080/producer/execution-plans,40,0,0,Diamond,3,301,5 15 | 1663859528445,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Diamond, 3, 301, 5 16 | 1663859528481,43,posting lineage plan,201,,simple post group 1-1,text,true,,300,657130,1,1,http://172.17.0.1:8080/producer/execution-plans,43,0,0,Diamond,3,351,5 17 | 1663859528543,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Diamond, 3, 351, 5 18 | 1663859528575,45,posting lineage plan,201,,simple post group 1-1,text,true,,300,750430,1,1,http://172.17.0.1:8080/producer/execution-plans,45,0,0,Diamond,3,401,5 19 | 1663859528632,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Diamond, 3, 401, 5 20 | 1663859528667,59,posting lineage plan,201,,simple post group 1-1,text,true,,300,843730,1,1,http://172.17.0.1:8080/producer/execution-plans,59,0,0,Diamond,3,451,5 21 | 1663859528739,5,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,5,0,0,Diamond, 3, 451, 5 22 | 1663859528778,56,posting lineage plan,201,,simple post group 1-1,text,true,,300,937030,1,1,http://172.17.0.1:8080/producer/execution-plans,56,0,0,Diamond,3,501,5 23 | 1663859528848,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Diamond, 3, 501, 5 24 | -------------------------------------------------------------------------------- /spline-testing/jmetered-spline-testing/reference/triangle-lineage-1-501@50reads-3ops-4attr.json.txt.res: -------------------------------------------------------------------------------- 1 | timeStamp,elapsed,label,responseCode,responseMessage,threadName,dataType,success,failureMessage,bytes,sentBytes,grpThreads,allThreads,URL,Latency,IdleTime,Connect,"graphType","operationCount","attributeCount","readCount" 2 | 1663859531898,49,posting lineage plan,201,,simple post group 1-1,text,true,,300,9288,1,1,http://172.17.0.1:8080/producer/execution-plans,48,0,31,Triangle,3,4,1 3 | 1663859532084,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Triangle, 3, 4, 1 4 | 1663859532101,36,posting lineage plan,201,,simple post group 1-1,text,true,,300,137915,1,1,http://172.17.0.1:8080/producer/execution-plans,36,0,0,Triangle,3,4,51 5 | 1663859532145,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Triangle, 3, 4, 51 6 | 1663859532174,76,posting lineage plan,201,,simple post group 1-1,text,true,,300,270625,1,1,http://172.17.0.1:8080/producer/execution-plans,76,0,0,Triangle,3,4,101 7 | 1663859532262,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Triangle, 3, 4, 101 8 | 1663859532281,69,posting lineage plan,201,,simple post group 1-1,text,true,,300,403526,1,1,http://172.17.0.1:8080/producer/execution-plans,69,0,0,Triangle,3,4,151 9 | 1663859532363,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Triangle, 3, 4, 151 10 | 1663859532386,86,posting lineage plan,201,,simple post group 1-1,text,true,,300,536426,1,1,http://172.17.0.1:8080/producer/execution-plans,85,0,0,Triangle,3,4,201 11 | 1663859532484,8,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,8,0,0,Triangle, 3, 4, 201 12 | 1663859532514,88,posting lineage plan,201,,simple post group 1-1,text,true,,300,669326,1,1,http://172.17.0.1:8080/producer/execution-plans,88,0,0,Triangle,3,4,251 13 | 1663859532622,5,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,5,0,0,Triangle, 3, 4, 251 14 | 1663859532650,92,posting lineage plan,201,,simple post group 1-1,text,true,,300,802226,1,1,http://172.17.0.1:8080/producer/execution-plans,92,0,0,Triangle,3,4,301 15 | 1663859532756,5,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,5,0,0,Triangle, 3, 4, 301 16 | 1663859532791,105,posting lineage plan,201,,simple post group 1-1,text,true,,300,935126,1,1,http://172.17.0.1:8080/producer/execution-plans,105,0,0,Triangle,3,4,351 17 | 1663859532912,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Triangle, 3, 4, 351 18 | 1663859532944,130,posting lineage plan,201,,simple post group 1-1,text,true,,300,1068027,1,1,http://172.17.0.1:8080/producer/execution-plans,129,0,0,Triangle,3,4,401 19 | 1663859533092,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Triangle, 3, 4, 401 20 | 1663859533131,144,posting lineage plan,201,,simple post group 1-1,text,true,,300,1200927,1,1,http://172.17.0.1:8080/producer/execution-plans,144,0,0,Triangle,3,4,451 21 | 1663859533302,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Triangle, 3, 4, 451 22 | 1663859533340,156,posting lineage plan,201,,simple post group 1-1,text,true,,300,1333827,1,1,http://172.17.0.1:8080/producer/execution-plans,156,0,0,Triangle,3,4,501 23 | 1663859533526,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Triangle, 3, 4, 501 24 | -------------------------------------------------------------------------------- /spline-testing/jmetered-spline-testing/reference/triangle-lineage-5reads-1-501@50ops-4attr.json.txt.res: -------------------------------------------------------------------------------- 1 | timeStamp,elapsed,label,responseCode,responseMessage,threadName,dataType,success,failureMessage,bytes,sentBytes,grpThreads,allThreads,URL,Latency,IdleTime,Connect,"graphType","operationCount","attributeCount","readCount" 2 | 1663859536556,57,posting lineage plan,201,,simple post group 1-1,text,true,,300,14859,1,1,http://172.17.0.1:8080/producer/execution-plans,56,0,36,Triangle,1,4,5 3 | 1663859536774,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Triangle, 1, 4, 5 4 | 1663859536790,21,posting lineage plan,201,,simple post group 1-1,text,true,,300,132567,1,1,http://172.17.0.1:8080/producer/execution-plans,21,0,0,Triangle,51,4,5 5 | 1663859536819,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Triangle, 51, 4, 5 6 | 1663859536843,28,posting lineage plan,201,,simple post group 1-1,text,true,,300,258319,1,1,http://172.17.0.1:8080/producer/execution-plans,27,0,0,Triangle,101,4,5 7 | 1663859536886,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Triangle, 101, 4, 5 8 | 1663859536908,34,posting lineage plan,201,,simple post group 1-1,text,true,,300,384070,1,1,http://172.17.0.1:8080/producer/execution-plans,34,0,0,Triangle,151,4,5 9 | 1663859536950,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Triangle, 151, 4, 5 10 | 1663859536975,38,posting lineage plan,201,,simple post group 1-1,text,true,,300,509820,1,1,http://172.17.0.1:8080/producer/execution-plans,37,0,0,Triangle,201,4,5 11 | 1663859537021,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Triangle, 201, 4, 5 12 | 1663859537046,53,posting lineage plan,201,,simple post group 1-1,text,true,,300,635570,1,1,http://172.17.0.1:8080/producer/execution-plans,52,0,0,Triangle,251,4,5 13 | 1663859537109,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Triangle, 251, 4, 5 14 | 1663859537138,50,posting lineage plan,201,,simple post group 1-1,text,true,,300,761320,1,1,http://172.17.0.1:8080/producer/execution-plans,50,0,0,Triangle,301,4,5 15 | 1663859537208,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Triangle, 301, 4, 5 16 | 1663859537241,58,posting lineage plan,201,,simple post group 1-1,text,true,,300,887070,1,1,http://172.17.0.1:8080/producer/execution-plans,58,0,0,Triangle,351,4,5 17 | 1663859537313,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Triangle, 351, 4, 5 18 | 1663859537348,61,posting lineage plan,201,,simple post group 1-1,text,true,,300,1012821,1,1,http://172.17.0.1:8080/producer/execution-plans,61,0,0,Triangle,401,4,5 19 | 1663859537425,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Triangle, 401, 4, 5 20 | 1663859537461,77,posting lineage plan,201,,simple post group 1-1,text,true,,300,1138571,1,1,http://172.17.0.1:8080/producer/execution-plans,77,0,0,Triangle,451,4,5 21 | 1663859537558,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Triangle, 451, 4, 5 22 | 1663859537601,79,posting lineage plan,201,,simple post group 1-1,text,true,,300,1264321,1,1,http://172.17.0.1:8080/producer/execution-plans,79,0,0,Triangle,501,4,5 23 | 1663859537705,8,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,8,0,0,Triangle, 501, 4, 5 24 | -------------------------------------------------------------------------------- /spline-testing/jmetered-spline-testing/reference/triangle-lineage-5reads-3-10@10ops-4attr.json.txt.res: -------------------------------------------------------------------------------- 1 | timeStamp,elapsed,label,responseCode,responseMessage,threadName,dataType,success,failureMessage,bytes,sentBytes,grpThreads,allThreads,URL,Latency,IdleTime,Connect,"graphType","operationCount","attributeCount","readCount" 2 | 1663859540831,52,posting lineage plan,201,,simple post group 1-1,text,true,,300,15889,1,1,http://172.17.0.1:8080/producer/execution-plans,52,0,32,Triangle,3,4,5 3 | 1663859541033,7,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,7,0,0,Triangle, 3, 4, 5 4 | -------------------------------------------------------------------------------- /spline-testing/jmetered-spline-testing/reference/triangle-lineage-5reads-3ops-1-501@50attr.json.txt.res: -------------------------------------------------------------------------------- 1 | timeStamp,elapsed,label,responseCode,responseMessage,threadName,dataType,success,failureMessage,bytes,sentBytes,grpThreads,allThreads,URL,Latency,IdleTime,Connect,"graphType","operationCount","attributeCount","readCount" 2 | 1663859544154,52,posting lineage plan,201,,simple post group 1-1,text,true,,300,5706,1,1,http://172.17.0.1:8080/producer/execution-plans,51,0,35,Triangle,3,1,5 3 | 1663859544354,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Triangle, 3, 1, 5 4 | 1663859544371,79,posting lineage plan,201,,simple post group 1-1,text,true,,300,175575,1,1,http://172.17.0.1:8080/producer/execution-plans,79,0,0,Triangle,3,51,5 5 | 1663859544459,5,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,5,0,0,Triangle, 3, 51, 5 6 | 1663859544485,28,posting lineage plan,201,,simple post group 1-1,text,true,,300,345537,1,1,http://172.17.0.1:8080/producer/execution-plans,28,0,0,Triangle,3,101,5 7 | 1663859544520,5,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,5,0,0,Triangle, 3, 101, 5 8 | 1663859544545,36,posting lineage plan,201,,simple post group 1-1,text,true,,300,515738,1,1,http://172.17.0.1:8080/producer/execution-plans,36,0,0,Triangle,3,151,5 9 | 1663859544590,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Triangle, 3, 151, 5 10 | 1663859544621,46,posting lineage plan,201,,simple post group 1-1,text,true,,300,685938,1,1,http://172.17.0.1:8080/producer/execution-plans,46,0,0,Triangle,3,201,5 11 | 1663859544679,5,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,5,0,0,Triangle, 3, 201, 5 12 | 1663859544715,56,posting lineage plan,201,,simple post group 1-1,text,true,,300,856138,1,1,http://172.17.0.1:8080/producer/execution-plans,56,0,0,Triangle,3,251,5 13 | 1663859544793,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Triangle, 3, 251, 5 14 | 1663859544827,62,posting lineage plan,201,,simple post group 1-1,text,true,,300,1026339,1,1,http://172.17.0.1:8080/producer/execution-plans,61,0,0,Triangle,3,301,5 15 | 1663859544904,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Triangle, 3, 301, 5 16 | 1663859544942,71,posting lineage plan,201,,simple post group 1-1,text,true,,300,1196539,1,1,http://172.17.0.1:8080/producer/execution-plans,71,0,0,Triangle,3,351,5 17 | 1663859545031,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Triangle, 3, 351, 5 18 | 1663859545071,92,posting lineage plan,201,,simple post group 1-1,text,true,,300,1366739,1,1,http://172.17.0.1:8080/producer/execution-plans,92,0,0,Triangle,3,401,5 19 | 1663859545189,5,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,5,0,0,Triangle, 3, 401, 5 20 | 1663859545237,94,posting lineage plan,201,,simple post group 1-1,text,true,,300,1536939,1,1,http://172.17.0.1:8080/producer/execution-plans,93,0,0,Triangle,3,451,5 21 | 1663859545363,6,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,6,0,0,Triangle, 3, 451, 5 22 | 1663859545410,113,posting lineage plan,201,,simple post group 1-1,text,true,,300,1707139,1,1,http://172.17.0.1:8080/producer/execution-plans,113,0,0,Triangle,3,501,5 23 | 1663859545561,5,posting lineage event,201,,simple post group 1-1,,true,,210,300,1,1,http://172.17.0.1:8080/producer/execution-events,5,0,0,Triangle, 3, 501, 5 24 | -------------------------------------------------------------------------------- /spline-testing/jmetered-spline-testing/requirements.txt: -------------------------------------------------------------------------------- 1 | docker==6.0.0 2 | docker-compose==1.29.2 3 | matplotlib==3.6.0 4 | pandas==1.4.4 5 | setuptools==58.1.0 6 | GitPython==3.1.27 7 | -------------------------------------------------------------------------------- /spline-testing/jmetered-spline-testing/run_jmetered_spline.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import argparse 3 | import subprocess 4 | import docker 5 | import os 6 | import platform 7 | import pandas as pd 8 | import matplotlib.pyplot as plt 9 | import git 10 | 11 | GIT_SPLINE_URL = "https://github.com/AbsaOSS/spline.git" 12 | SPLINE_DEFAULT_BRANCH = "develop" 13 | 14 | SPLINE_CORE_VERSION = "1.0.0-SNAPSHOT" # needs to be in sync with .env 15 | CUSTOM_IMAGES = [ 16 | f"local-only/spline-testing/testing-spline-db-admin:{SPLINE_CORE_VERSION}", 17 | f"local-only/spline-testing/testing-spline-rest-server:{SPLINE_CORE_VERSION}" 18 | ] 19 | 20 | SPLINE_BUILD = "{mvn} install -DskipTests" 21 | 22 | JMETER_COLNAME_TIMESTAMP = "timeStamp" 23 | JMETER_COLNAME_ELAPSED = "elapsed" 24 | JMETER_COLNAME_LABEL = "label" 25 | # coming from Jmeter, but custom-added (graphType,operationCount,attributeCount,readCount) 26 | JMETER_COLNAME_GRAPH_TYPE = "graphType" 27 | JMETER_COLNAME_OP_COUNT = "operationCount" 28 | JMETER_COLNAME_ATTR_COUNT = "attributeCount" 29 | JMETER_COLNAME_READ_COUNT = "readCount" 30 | 31 | # script folders: 32 | RESULTS_FOLDER_NAME = "results" 33 | REFERENCE_FOLDER_NAME = "reference" 34 | PROCESSED_FOLDER_NAME = "processed_results" 35 | GRAPHS_FOLDER_NAME = "graphs" 36 | 37 | # Matplotlib: 38 | PLOT_MARKER = "o" 39 | PLOT_MARKER_SIZE = 2 40 | PLOT_DPI = 300 41 | 42 | 43 | def parse_args() -> argparse.Namespace: 44 | parser = argparse.ArgumentParser( 45 | prog='run_jmetered_spline', 46 | description='Spline backend simple testing (jmetered)', 47 | formatter_class=argparse.ArgumentDefaultsHelpFormatter # prints default values, too, on help (-h) 48 | ) 49 | 50 | parser.add_argument('-n', '--no-rebuild', action='store_true', dest="no_rebuild", default=False, 51 | help="if specified, spline will not be rebuilt (use when rerunning on an existing codebase)") 52 | # parser.add_argument('-v', '--verbose', action="store_true", default=False, 53 | # help="prints extra information while running.") 54 | 55 | parser.add_argument('-b', '--spline-branch', dest="spline_branch", default=SPLINE_DEFAULT_BRANCH, 56 | help="Name of spline branch to test") 57 | 58 | return parser.parse_args() 59 | 60 | 61 | def get_mvn_by_os() -> str: 62 | if platform.system() == "Windows": 63 | return "mvn.cmd" 64 | else: 65 | return "mvn" 66 | 67 | 68 | def build_spline(branch): 69 | spline_dir = f"{root_dir}/spline" 70 | 71 | if not os.path.exists(spline_dir): 72 | print(f"Cloning Spline into {spline_dir} (branch {branch})") 73 | git.Repo.clone_from(GIT_SPLINE_URL, spline_dir, branch=branch) 74 | else: 75 | print(f"Pulling Spline into {spline_dir} (branch {branch})") 76 | repo = git.Repo(spline_dir) 77 | repo.git.checkout(branch) 78 | repo.remotes[0].pull() 79 | 80 | os.chdir(f"{root_dir}/spline") 81 | print(f"Current working directory: {os.getcwd()}") 82 | 83 | mvn = get_mvn_by_os() 84 | spline_build_command = SPLINE_BUILD.format(mvn=mvn) 85 | print(f"Building spline via '{spline_build_command}'") 86 | subprocess.run(spline_build_command, shell=True, check=True) 87 | 88 | 89 | def run_docker_compose(): 90 | os.chdir(root_dir) 91 | os.makedirs(f"{root_dir}/{RESULTS_FOLDER_NAME}", exist_ok=True) 92 | # --exit-code-from = reports exit code from this container 93 | # AND implies '--abort-on-container-exit' - will 'docker-compose down' after any container has exited 94 | subprocess.run("docker-compose up --exit-code-from jmeter --build", shell=True, check=True) 95 | print("docker-compose up done") 96 | 97 | 98 | def cleanup_docker(): 99 | client = docker.from_env() 100 | 101 | for image_name in CUSTOM_IMAGES: 102 | print(f"Cleaning up custom image '{image_name}'") 103 | try: 104 | client.images.remove(image_name, force=True) 105 | print(" - done") 106 | except docker.errors.ImageNotFound as inf: 107 | print(f" - custom image custom image '{image_name}' not found!") 108 | 109 | print("docker-compose cleanup finished") 110 | 111 | 112 | def enrich_results_with_reference(): 113 | result_filenames = os.listdir(f"{root_dir}/{RESULTS_FOLDER_NAME}") 114 | os.makedirs(f"{root_dir}/{PROCESSED_FOLDER_NAME}", exist_ok=True) 115 | for result_filename in result_filenames: 116 | reference_df = pd.read_csv(f"./{REFERENCE_FOLDER_NAME}/{result_filename}") 117 | reference_df[JMETER_COLNAME_LABEL] = reference_df[JMETER_COLNAME_LABEL].map(lambda x: f"reference {x}") # in-place 118 | normalized_reference_df = normalize_dataframe_timestamp(reference_df) 119 | 120 | results_df = pd.read_csv(f"./{RESULTS_FOLDER_NAME}/{result_filename}") 121 | normalized_results_df = normalize_dataframe_timestamp(results_df) 122 | 123 | joined_df = pd.concat([normalized_reference_df, normalized_results_df]) 124 | 125 | joined_df.to_csv(f"./{PROCESSED_FOLDER_NAME}/{result_filename}", index=False) 126 | 127 | 128 | def normalize_dataframe_timestamp(df: pd.core.frame.DataFrame) -> pd.core.frame.DataFrame: 129 | min_ts = df[JMETER_COLNAME_TIMESTAMP].min() 130 | normalized_df = df.copy() # deep copy for the fn to behave immutably 131 | normalized_df[JMETER_COLNAME_TIMESTAMP] = normalized_df[JMETER_COLNAME_TIMESTAMP].map(lambda x: x - min_ts) 132 | return normalized_df 133 | 134 | 135 | def divide_df_ref_and_non_ref(df: pd.core.frame.DataFrame) -> list[pd.core.frame.DataFrame]: 136 | ref_df = df[df['label'].str.startswith("reference")] 137 | nonref_df = df[~df['label'].str.startswith("reference")] 138 | 139 | return [ref_df, nonref_df] 140 | 141 | 142 | def divide_plan_and_event(df: pd.core.frame.DataFrame) -> list[pd.core.frame.DataFrame]: 143 | plan_df = df[df['label'].str.endswith("plan")] 144 | non_plan_df = df[~df['label'].str.endswith("plan")] 145 | 146 | return [plan_df, non_plan_df] 147 | 148 | 149 | def find_variable_column_name(df: pd.core.frame.DataFrame) -> str: 150 | unique_ops, unique_attrs, unique_reads = df[[JMETER_COLNAME_OP_COUNT, JMETER_COLNAME_ATTR_COUNT, JMETER_COLNAME_READ_COUNT]].nunique().values.tolist() 151 | # debug print(f"uniques: {unique_ops}, {unique_attrs}, {unique_reads}") 152 | 153 | if unique_ops > 1: 154 | print(f" Using variable column '{JMETER_COLNAME_OP_COUNT}' (nuniques = {unique_ops})") 155 | return JMETER_COLNAME_OP_COUNT 156 | elif unique_attrs > 1: 157 | print(f" Using variable column '{JMETER_COLNAME_ATTR_COUNT}' (nuniques = {unique_attrs})") 158 | return JMETER_COLNAME_ATTR_COUNT 159 | elif unique_reads > 1: 160 | print(f" Using variable column '{JMETER_COLNAME_READ_COUNT}' (nuniques = {unique_reads})") 161 | return JMETER_COLNAME_READ_COUNT 162 | else: 163 | print(f" Fall-backing variable column to '{JMETER_COLNAME_TIMESTAMP}'") 164 | return JMETER_COLNAME_TIMESTAMP 165 | 166 | 167 | def generate_graph_from_processed_result(result_filename: str): 168 | 169 | # rereading the results file, because we want the graph drawing process to be independent from the processing part 170 | all_data = pd.read_csv(f"./{PROCESSED_FOLDER_NAME}/{result_filename}") 171 | ref_df, res_df = divide_df_ref_and_non_ref(all_data) 172 | 173 | ref_plan_df, ref_event_df = divide_plan_and_event(ref_df) 174 | res_plan_df, res_event_df = divide_plan_and_event(res_df) 175 | 176 | var_colname = find_variable_column_name(all_data) # operationCount,attributeCount,readCount or (worst-case) timeStamp 177 | 178 | plt.clf() # clear previous state if any 179 | plt.yscale("log") # because initial values tend to be outliers 180 | 181 | ref_plan_elapsed = ref_plan_df[JMETER_COLNAME_ELAPSED].tolist() 182 | ref_plan_var = ref_plan_df[var_colname].tolist() 183 | plt.plot(ref_plan_var, ref_plan_elapsed, marker=PLOT_MARKER, markersize=PLOT_MARKER_SIZE, label="Reference lineage plan posting") 184 | 185 | res_plan_elapsed = res_plan_df[JMETER_COLNAME_ELAPSED].tolist() 186 | res_plan_var = res_plan_df[var_colname].tolist() 187 | plt.plot(res_plan_var, res_plan_elapsed, marker=PLOT_MARKER, markersize=PLOT_MARKER_SIZE, label="Current lineage plan posting") 188 | 189 | ref_event_elapsed = ref_event_df[JMETER_COLNAME_ELAPSED].tolist() 190 | ref_event_var = ref_event_df[var_colname].tolist() 191 | plt.plot(ref_event_var, ref_event_elapsed, marker=PLOT_MARKER, markersize=PLOT_MARKER_SIZE, label="Reference lineage event posting") 192 | 193 | res_event_elapsed = res_event_df[JMETER_COLNAME_ELAPSED].tolist() 194 | res_event_var = res_event_df[var_colname].tolist() 195 | plt.plot(res_event_var, res_event_elapsed, marker=PLOT_MARKER, markersize=PLOT_MARKER_SIZE, label="Current lineage event posting") 196 | 197 | plt.xlabel(f"Variable '{var_colname}'") 198 | plt.ylabel('Elapsed time [ms]') 199 | plt.title(f"Elapsed time dependence on variable '{var_colname}'") 200 | plt.legend() 201 | 202 | # debug: 203 | # print(f"ref_plan_var={ref_plan_var}") 204 | # print(f"res_plan_var={res_plan_var}") 205 | # print(f"ref_event_var={ref_event_var}") 206 | # print(f"res_event_var={res_event_var}") 207 | 208 | plt.savefig(f"{root_dir}/{GRAPHS_FOLDER_NAME}/{result_filename}.png", dpi=PLOT_DPI) 209 | 210 | 211 | def generate_graphs(): 212 | result_filenames = os.listdir(f"{root_dir}/{PROCESSED_FOLDER_NAME}") 213 | os.makedirs(f"{root_dir}/{GRAPHS_FOLDER_NAME}", exist_ok=True) 214 | 215 | # rereading the results file, because we want the graph drawing process to be independent from the processing part 216 | for result_filename in result_filenames: 217 | print(f"Generating graph for file {PROCESSED_FOLDER_NAME}/{result_filename}") 218 | generate_graph_from_processed_result(result_filename) 219 | 220 | 221 | def run(parsed_args: argparse.Namespace): 222 | spline_branch = parsed_args.spline_branch 223 | no_rebuild = parsed_args.no_rebuild 224 | 225 | if not no_rebuild: 226 | build_spline(spline_branch) 227 | 228 | run_docker_compose() 229 | 230 | enrich_results_with_reference() 231 | generate_graphs() 232 | 233 | cleanup_docker() 234 | 235 | print("All testing done.") 236 | 237 | 238 | if __name__ == '__main__': 239 | args = parse_args() 240 | 241 | # globals script vars 242 | root_dir = os.getcwd() 243 | 244 | run(args) 245 | -------------------------------------------------------------------------------- /spline-testing/jmetered-spline-testing/tests/triangle-lineage-5reads-3-10@10ops-4attr.json.txt: -------------------------------------------------------------------------------- 1 | {"id":"8e3815c6-c3c8-4ad5-bda1-52020a8f864f","name":"generated plan 8e3815c6-c3c8-4ad5-bda1-52020a8f864f","labels":{},"operations":{"write":{"outputSource":"file://splinegen/write.csv","append":false,"id":"470ca18b-964d-4c21-8dba-19da8c86f98e","name":"generatedWrite","childIds":["f2051b0f-0881-4a2b-a8f8-84c692a44a76"],"params":{},"extra":{}},"reads":[{"inputSources":["file://splinegen/read_1.csv"],"id":"1","name":"generated read 1","output":["7e735357-da99-4e68-b509-396ea4a67f07","4be5f124-34b9-4009-8f98-be1001572ad5","0f85d3a1-1491-4c4a-b004-ebaaf52dcd6c","840ea028-3353-42b0-8e83-1c80f35efbc4"],"params":{},"extra":{}},{"inputSources":["file://splinegen/read_2.csv"],"id":"2","name":"generated read 2","output":["7e5fdb95-9cf9-4bb8-8232-0eb13a064962","14f8933f-4bbc-4194-b0a3-7ceb86e0185b","5dc009f2-18a7-45dc-bf3b-f116c514af07","59d24fee-8255-49d9-840a-843b7e637d27"],"params":{},"extra":{}},{"inputSources":["file://splinegen/read_3.csv"],"id":"3","name":"generated read 3","output":["7b0f89aa-7a97-4197-acc5-dd5a2b4b3768","43a2ccb4-2e67-473b-9bbf-7864e5a8ab18","7c6c25d0-482a-4a7f-984f-08746aaed084","f68fcf4f-5649-45a9-b554-b8c68f145319"],"params":{},"extra":{}},{"inputSources":["file://splinegen/read_4.csv"],"id":"4","name":"generated read 4","output":["2ea434a9-6fa3-4fda-b013-72fa83462924","6a911c1c-c9d5-4580-9383-859e191ed289","94eaeb3b-e174-4ddd-a0d0-4f5cbce34547","0e4f5f77-4b93-4b0d-af54-825d2458a4fc"],"params":{},"extra":{}},{"inputSources":["file://splinegen/read_5.csv"],"id":"5","name":"generated read 5","output":["c33a09e5-00e8-46de-9f7f-1a65e91effab","7c586341-7ac1-4d1e-8b43-b5f858e53079","7a8bcb48-b8ec-4237-b627-02b0998f88d7","b126009d-c708-4d2a-966e-e9f62a4f1751"],"params":{},"extra":{}}],"other":[{"id":"c81d64f8-e568-4227-84ca-efb8629240de","name":"generated data operation c81d64f8-e568-4227-84ca-efb8629240de","childIds":["1"],"output":["e3f4bd51-0be1-433b-ad8a-db8cfa0913c5","aea0440b-f328-4cd2-91dc-d0b65db9762c","be5e162d-20b8-4838-bfdf-bd445b699885","9e9bb73a-1b7b-4914-9d71-136e5257b11b"],"params":{},"extra":{}},{"id":"49b15600-7837-4887-9e81-83df3bacb858","name":"generated data operation 49b15600-7837-4887-9e81-83df3bacb858","childIds":["2"],"output":["8b166a68-2efa-4557-a7f3-360312eb4dfc","de013ca7-4a64-428f-a9e2-65ac91ed8ab4","74fab2fc-ccd7-46e2-b261-95e82e906838","0a57ec56-6a5c-4ab1-b229-26579ac54328"],"params":{},"extra":{}},{"id":"ef5f1111-2390-4e89-a2dc-807b70881575","name":"generated data operation ef5f1111-2390-4e89-a2dc-807b70881575","childIds":["3","4","5"],"output":["a0609431-d785-41b6-944c-e72d1d2b39da","3ef5ee74-dc5a-4b93-be35-6c9e28dec283","1bc2f35b-fc90-4da2-8992-56c8b30c220f","54d580c0-f113-4214-900f-ec5231cd1b45"],"params":{},"extra":{}},{"id":"f2051b0f-0881-4a2b-a8f8-84c692a44a76","name":"generated data operation f2051b0f-0881-4a2b-a8f8-84c692a44a76","childIds":["c81d64f8-e568-4227-84ca-efb8629240de","49b15600-7837-4887-9e81-83df3bacb858","ef5f1111-2390-4e89-a2dc-807b70881575"],"output":["e3f4bd51-0be1-433b-ad8a-db8cfa0913c5","aea0440b-f328-4cd2-91dc-d0b65db9762c","be5e162d-20b8-4838-bfdf-bd445b699885","9e9bb73a-1b7b-4914-9d71-136e5257b11b","8b166a68-2efa-4557-a7f3-360312eb4dfc","de013ca7-4a64-428f-a9e2-65ac91ed8ab4","74fab2fc-ccd7-46e2-b261-95e82e906838","0a57ec56-6a5c-4ab1-b229-26579ac54328","a0609431-d785-41b6-944c-e72d1d2b39da","3ef5ee74-dc5a-4b93-be35-6c9e28dec283","1bc2f35b-fc90-4da2-8992-56c8b30c220f","54d580c0-f113-4214-900f-ec5231cd1b45"],"params":{},"extra":{}}]},"attributes":[{"id":"6a911c1c-c9d5-4580-9383-859e191ed289","childRefs":[],"extra":{},"name":"dummy_attr_2"},{"id":"2ea434a9-6fa3-4fda-b013-72fa83462924","childRefs":[],"extra":{},"name":"dummy_attr_1"},{"id":"0a57ec56-6a5c-4ab1-b229-26579ac54328","childRefs":[{"__exprId":"4b1a1d4c-b08e-4f18-9c56-efeaef541bd0"}],"extra":{},"name":"dummy_attr_0a57ec56-6a5c-4ab1-b229-26579ac54328"},{"id":"840ea028-3353-42b0-8e83-1c80f35efbc4","childRefs":[],"extra":{},"name":"dummy_attr_4"},{"id":"54d580c0-f113-4214-900f-ec5231cd1b45","childRefs":[{"__exprId":"4d503e94-f46b-4442-9840-d472189ae154"}],"extra":{},"name":"dummy_attr_54d580c0-f113-4214-900f-ec5231cd1b45"},{"id":"9e9bb73a-1b7b-4914-9d71-136e5257b11b","childRefs":[{"__exprId":"44352a05-db80-4c57-b174-11918e695488"}],"extra":{},"name":"dummy_attr_9e9bb73a-1b7b-4914-9d71-136e5257b11b"},{"id":"0e4f5f77-4b93-4b0d-af54-825d2458a4fc","childRefs":[],"extra":{},"name":"dummy_attr_4"},{"id":"7b0f89aa-7a97-4197-acc5-dd5a2b4b3768","childRefs":[],"extra":{},"name":"dummy_attr_1"},{"id":"a0609431-d785-41b6-944c-e72d1d2b39da","childRefs":[{"__exprId":"6b565a1e-a67c-4166-b678-b8ece7ded9ec"}],"extra":{},"name":"dummy_attr_a0609431-d785-41b6-944c-e72d1d2b39da"},{"id":"b126009d-c708-4d2a-966e-e9f62a4f1751","childRefs":[],"extra":{},"name":"dummy_attr_4"},{"id":"7e5fdb95-9cf9-4bb8-8232-0eb13a064962","childRefs":[],"extra":{},"name":"dummy_attr_1"},{"id":"59d24fee-8255-49d9-840a-843b7e637d27","childRefs":[],"extra":{},"name":"dummy_attr_4"},{"id":"f68fcf4f-5649-45a9-b554-b8c68f145319","childRefs":[],"extra":{},"name":"dummy_attr_4"},{"id":"43a2ccb4-2e67-473b-9bbf-7864e5a8ab18","childRefs":[],"extra":{},"name":"dummy_attr_2"},{"id":"3ef5ee74-dc5a-4b93-be35-6c9e28dec283","childRefs":[{"__exprId":"8306b2ff-ed6a-48cb-bd5d-73eef98deb2f"}],"extra":{},"name":"dummy_attr_3ef5ee74-dc5a-4b93-be35-6c9e28dec283"},{"id":"1b618e7c-6964-45f7-947d-d6946545840e","childRefs":[{"__exprId":"ae90d5e2-10df-4de9-936b-0c2c6c43a330"}],"extra":{},"name":"dummy_attr_1b618e7c-6964-45f7-947d-d6946545840e"},{"id":"7a8bcb48-b8ec-4237-b627-02b0998f88d7","childRefs":[],"extra":{},"name":"dummy_attr_3"},{"id":"14f8933f-4bbc-4194-b0a3-7ceb86e0185b","childRefs":[],"extra":{},"name":"dummy_attr_2"},{"id":"7c586341-7ac1-4d1e-8b43-b5f858e53079","childRefs":[],"extra":{},"name":"dummy_attr_2"},{"id":"74fab2fc-ccd7-46e2-b261-95e82e906838","childRefs":[{"__exprId":"1025c942-ea3a-4b1f-aaec-b8734823df6d"}],"extra":{},"name":"dummy_attr_74fab2fc-ccd7-46e2-b261-95e82e906838"},{"id":"8b166a68-2efa-4557-a7f3-360312eb4dfc","childRefs":[{"__exprId":"dbe62b3f-45ea-47ff-ad8c-ec276fb9e750"}],"extra":{},"name":"dummy_attr_8b166a68-2efa-4557-a7f3-360312eb4dfc"},{"id":"5826c00a-abd0-4552-a75f-c0497aabe492","childRefs":[{"__exprId":"0bedd897-5b61-4b0e-859e-dc0e9f078411"}],"extra":{},"name":"dummy_attr_5826c00a-abd0-4552-a75f-c0497aabe492"},{"id":"be5e162d-20b8-4838-bfdf-bd445b699885","childRefs":[{"__exprId":"a0b12d7d-3a34-4554-b48f-554bc0f9bd08"}],"extra":{},"name":"dummy_attr_be5e162d-20b8-4838-bfdf-bd445b699885"},{"id":"bcada20b-ce9e-46ed-862a-6ed629bc616f","childRefs":[{"__exprId":"2ff5bcce-3868-4a35-bb75-20d82318a7b7"}],"extra":{},"name":"dummy_attr_bcada20b-ce9e-46ed-862a-6ed629bc616f"},{"id":"eda7b853-df27-48b7-a84b-63929df44f63","childRefs":[{"__exprId":"c243ce1c-1cad-4043-ac26-30e2e438c2c2"}],"extra":{},"name":"dummy_attr_eda7b853-df27-48b7-a84b-63929df44f63"},{"id":"7e735357-da99-4e68-b509-396ea4a67f07","childRefs":[],"extra":{},"name":"dummy_attr_1"},{"id":"4be5f124-34b9-4009-8f98-be1001572ad5","childRefs":[],"extra":{},"name":"dummy_attr_2"},{"id":"dbfe0c89-7660-4b37-8ab1-29c2311bfc20","childRefs":[{"__exprId":"d1420f02-1c58-46c8-ae2b-e097fb58f836"}],"extra":{},"name":"dummy_attr_dbfe0c89-7660-4b37-8ab1-29c2311bfc20"},{"id":"94eaeb3b-e174-4ddd-a0d0-4f5cbce34547","childRefs":[],"extra":{},"name":"dummy_attr_3"},{"id":"1bc2f35b-fc90-4da2-8992-56c8b30c220f","childRefs":[{"__exprId":"827a0d36-01b9-439a-ad3a-90f9209d0b4d"}],"extra":{},"name":"dummy_attr_1bc2f35b-fc90-4da2-8992-56c8b30c220f"},{"id":"e3f4bd51-0be1-433b-ad8a-db8cfa0913c5","childRefs":[{"__exprId":"02b66e61-ef51-4db2-b2f3-794d89487fc4"}],"extra":{},"name":"dummy_attr_e3f4bd51-0be1-433b-ad8a-db8cfa0913c5"},{"id":"35c266ec-0238-4a1a-98de-5d1842c69703","childRefs":[{"__exprId":"ad764fcc-48e5-4c4a-89e4-8b81cbcacade"}],"extra":{},"name":"dummy_attr_35c266ec-0238-4a1a-98de-5d1842c69703"},{"id":"0f85d3a1-1491-4c4a-b004-ebaaf52dcd6c","childRefs":[],"extra":{},"name":"dummy_attr_3"},{"id":"c47969d1-448f-48e8-aaf2-edf4f6c694af","childRefs":[{"__exprId":"664f4e27-e248-435f-8a29-77c711fdb454"}],"extra":{},"name":"dummy_attr_c47969d1-448f-48e8-aaf2-edf4f6c694af"},{"id":"7c6c25d0-482a-4a7f-984f-08746aaed084","childRefs":[],"extra":{},"name":"dummy_attr_3"},{"id":"e0d47b9b-2121-4765-8751-6dd6eb290453","childRefs":[{"__exprId":"33eed365-e89d-46dd-8fac-21040393c0b1"}],"extra":{},"name":"dummy_attr_e0d47b9b-2121-4765-8751-6dd6eb290453"},{"id":"de013ca7-4a64-428f-a9e2-65ac91ed8ab4","childRefs":[{"__exprId":"1090a4b2-94d9-42a1-94c6-435f2add0777"}],"extra":{},"name":"dummy_attr_de013ca7-4a64-428f-a9e2-65ac91ed8ab4"},{"id":"5dc009f2-18a7-45dc-bf3b-f116c514af07","childRefs":[],"extra":{},"name":"dummy_attr_3"},{"id":"aea0440b-f328-4cd2-91dc-d0b65db9762c","childRefs":[{"__exprId":"a353b5db-08d2-43ae-900b-e33cf7f8eb33"}],"extra":{},"name":"dummy_attr_aea0440b-f328-4cd2-91dc-d0b65db9762c"},{"id":"c33a09e5-00e8-46de-9f7f-1a65e91effab","childRefs":[],"extra":{},"name":"dummy_attr_1"}],"expressions":{"functions":[{"id":"02b66e61-ef51-4db2-b2f3-794d89487fc4","childRefs":[{"__exprId":"09d988e4-8018-49c1-801c-ae5b65465dc7"},{"__attrId":"7e735357-da99-4e68-b509-396ea4a67f07"}],"extra":{},"name":"dummy_funcexpr","params":{}},{"id":"a353b5db-08d2-43ae-900b-e33cf7f8eb33","childRefs":[{"__exprId":"e01f09be-c3b7-442d-b430-a7345adca071"},{"__attrId":"4be5f124-34b9-4009-8f98-be1001572ad5"}],"extra":{},"name":"dummy_funcexpr","params":{}},{"id":"a0b12d7d-3a34-4554-b48f-554bc0f9bd08","childRefs":[{"__exprId":"562d0c51-a5bb-4911-b740-f2c83228f391"},{"__attrId":"0f85d3a1-1491-4c4a-b004-ebaaf52dcd6c"}],"extra":{},"name":"dummy_funcexpr","params":{}},{"id":"44352a05-db80-4c57-b174-11918e695488","childRefs":[{"__exprId":"d6f478a8-92e2-446b-a5a1-4cb8cfd7d721"},{"__attrId":"840ea028-3353-42b0-8e83-1c80f35efbc4"}],"extra":{},"name":"dummy_funcexpr","params":{}},{"id":"dbe62b3f-45ea-47ff-ad8c-ec276fb9e750","childRefs":[{"__exprId":"15d6a454-6517-42eb-9fef-bb834b51a931"},{"__attrId":"7e5fdb95-9cf9-4bb8-8232-0eb13a064962"}],"extra":{},"name":"dummy_funcexpr","params":{}},{"id":"1090a4b2-94d9-42a1-94c6-435f2add0777","childRefs":[{"__exprId":"97158ce5-4ec2-4abb-b00f-d71dc0529a57"},{"__attrId":"14f8933f-4bbc-4194-b0a3-7ceb86e0185b"}],"extra":{},"name":"dummy_funcexpr","params":{}},{"id":"1025c942-ea3a-4b1f-aaec-b8734823df6d","childRefs":[{"__exprId":"edc78389-f3df-461a-878a-944e0d4a5837"},{"__attrId":"5dc009f2-18a7-45dc-bf3b-f116c514af07"}],"extra":{},"name":"dummy_funcexpr","params":{}},{"id":"4b1a1d4c-b08e-4f18-9c56-efeaef541bd0","childRefs":[{"__exprId":"ba0e238b-f258-4e08-9be4-86674c1b31ec"},{"__attrId":"59d24fee-8255-49d9-840a-843b7e637d27"}],"extra":{},"name":"dummy_funcexpr","params":{}},{"id":"6b565a1e-a67c-4166-b678-b8ece7ded9ec","childRefs":[{"__exprId":"0c1628d8-4a97-44c3-8ddd-6e2d492d0d99"},{"__attrId":"7b0f89aa-7a97-4197-acc5-dd5a2b4b3768"}],"extra":{},"name":"dummy_funcexpr","params":{}},{"id":"8306b2ff-ed6a-48cb-bd5d-73eef98deb2f","childRefs":[{"__exprId":"eac9516d-2c89-4348-82b1-29bbaa811f4c"},{"__attrId":"43a2ccb4-2e67-473b-9bbf-7864e5a8ab18"}],"extra":{},"name":"dummy_funcexpr","params":{}},{"id":"827a0d36-01b9-439a-ad3a-90f9209d0b4d","childRefs":[{"__exprId":"098c4a9f-08a3-44ab-aae0-4d930b724d47"},{"__attrId":"7c6c25d0-482a-4a7f-984f-08746aaed084"}],"extra":{},"name":"dummy_funcexpr","params":{}},{"id":"4d503e94-f46b-4442-9840-d472189ae154","childRefs":[{"__exprId":"2cb045cf-983a-4edc-8da4-bb6ba5af043a"},{"__attrId":"f68fcf4f-5649-45a9-b554-b8c68f145319"}],"extra":{},"name":"dummy_funcexpr","params":{}},{"id":"ae90d5e2-10df-4de9-936b-0c2c6c43a330","childRefs":[{"__exprId":"64017e45-47d9-4aa7-b8dc-c6860f415f7e"},{"__attrId":"2ea434a9-6fa3-4fda-b013-72fa83462924"}],"extra":{},"name":"dummy_funcexpr","params":{}},{"id":"664f4e27-e248-435f-8a29-77c711fdb454","childRefs":[{"__exprId":"1b5251a9-235b-459d-bbce-cda42f5ef43a"},{"__attrId":"6a911c1c-c9d5-4580-9383-859e191ed289"}],"extra":{},"name":"dummy_funcexpr","params":{}},{"id":"0bedd897-5b61-4b0e-859e-dc0e9f078411","childRefs":[{"__exprId":"013de7a6-f39e-4cbe-bbb3-9275754aacc5"},{"__attrId":"94eaeb3b-e174-4ddd-a0d0-4f5cbce34547"}],"extra":{},"name":"dummy_funcexpr","params":{}},{"id":"2ff5bcce-3868-4a35-bb75-20d82318a7b7","childRefs":[{"__exprId":"428b103a-2f50-4365-8743-9772b61f41b1"},{"__attrId":"0e4f5f77-4b93-4b0d-af54-825d2458a4fc"}],"extra":{},"name":"dummy_funcexpr","params":{}},{"id":"33eed365-e89d-46dd-8fac-21040393c0b1","childRefs":[{"__exprId":"ab594065-6b00-4d16-804f-b0a20f39e908"},{"__attrId":"c33a09e5-00e8-46de-9f7f-1a65e91effab"}],"extra":{},"name":"dummy_funcexpr","params":{}},{"id":"ad764fcc-48e5-4c4a-89e4-8b81cbcacade","childRefs":[{"__exprId":"68dbe5bd-d70e-4fa4-a459-d395b31951cb"},{"__attrId":"7c586341-7ac1-4d1e-8b43-b5f858e53079"}],"extra":{},"name":"dummy_funcexpr","params":{}},{"id":"c243ce1c-1cad-4043-ac26-30e2e438c2c2","childRefs":[{"__exprId":"be1e3b71-5f70-4184-bd86-64aef252354a"},{"__attrId":"7a8bcb48-b8ec-4237-b627-02b0998f88d7"}],"extra":{},"name":"dummy_funcexpr","params":{}},{"id":"d1420f02-1c58-46c8-ae2b-e097fb58f836","childRefs":[{"__exprId":"11f019af-65df-4c15-92d4-6121f103a31c"},{"__attrId":"b126009d-c708-4d2a-966e-e9f62a4f1751"}],"extra":{},"name":"dummy_funcexpr","params":{}}],"constants":[{"id":"09d988e4-8018-49c1-801c-ae5b65465dc7","extra":{},"value":"val_09d988e4-8018-49c1-801c-ae5b65465dc7"},{"id":"e01f09be-c3b7-442d-b430-a7345adca071","extra":{},"value":"val_e01f09be-c3b7-442d-b430-a7345adca071"},{"id":"562d0c51-a5bb-4911-b740-f2c83228f391","extra":{},"value":"val_562d0c51-a5bb-4911-b740-f2c83228f391"},{"id":"d6f478a8-92e2-446b-a5a1-4cb8cfd7d721","extra":{},"value":"val_d6f478a8-92e2-446b-a5a1-4cb8cfd7d721"},{"id":"15d6a454-6517-42eb-9fef-bb834b51a931","extra":{},"value":"val_15d6a454-6517-42eb-9fef-bb834b51a931"},{"id":"97158ce5-4ec2-4abb-b00f-d71dc0529a57","extra":{},"value":"val_97158ce5-4ec2-4abb-b00f-d71dc0529a57"},{"id":"edc78389-f3df-461a-878a-944e0d4a5837","extra":{},"value":"val_edc78389-f3df-461a-878a-944e0d4a5837"},{"id":"ba0e238b-f258-4e08-9be4-86674c1b31ec","extra":{},"value":"val_ba0e238b-f258-4e08-9be4-86674c1b31ec"},{"id":"0c1628d8-4a97-44c3-8ddd-6e2d492d0d99","extra":{},"value":"val_0c1628d8-4a97-44c3-8ddd-6e2d492d0d99"},{"id":"eac9516d-2c89-4348-82b1-29bbaa811f4c","extra":{},"value":"val_eac9516d-2c89-4348-82b1-29bbaa811f4c"},{"id":"098c4a9f-08a3-44ab-aae0-4d930b724d47","extra":{},"value":"val_098c4a9f-08a3-44ab-aae0-4d930b724d47"},{"id":"2cb045cf-983a-4edc-8da4-bb6ba5af043a","extra":{},"value":"val_2cb045cf-983a-4edc-8da4-bb6ba5af043a"},{"id":"64017e45-47d9-4aa7-b8dc-c6860f415f7e","extra":{},"value":"val_64017e45-47d9-4aa7-b8dc-c6860f415f7e"},{"id":"1b5251a9-235b-459d-bbce-cda42f5ef43a","extra":{},"value":"val_1b5251a9-235b-459d-bbce-cda42f5ef43a"},{"id":"013de7a6-f39e-4cbe-bbb3-9275754aacc5","extra":{},"value":"val_013de7a6-f39e-4cbe-bbb3-9275754aacc5"},{"id":"428b103a-2f50-4365-8743-9772b61f41b1","extra":{},"value":"val_428b103a-2f50-4365-8743-9772b61f41b1"},{"id":"ab594065-6b00-4d16-804f-b0a20f39e908","extra":{},"value":"val_ab594065-6b00-4d16-804f-b0a20f39e908"},{"id":"68dbe5bd-d70e-4fa4-a459-d395b31951cb","extra":{},"value":"val_68dbe5bd-d70e-4fa4-a459-d395b31951cb"},{"id":"be1e3b71-5f70-4184-bd86-64aef252354a","extra":{},"value":"val_be1e3b71-5f70-4184-bd86-64aef252354a"},{"id":"11f019af-65df-4c15-92d4-6121f103a31c","extra":{},"value":"val_11f019af-65df-4c15-92d4-6121f103a31c"}]},"systemInfo":{"name":"spline-data-gen","version":"1.0.0-SNAPSHOT"},"extraInfo":{"graph-type":"Triangle","operationCount":3,"attributeCount":4,"readCount":5}} 2 | [{"planId":"8e3815c6-c3c8-4ad5-bda1-52020a8f864f","labels":{},"timestamp":1660049194971,"extra":{}}] 3 | -------------------------------------------------------------------------------- /spline-testing/jmetered-spline-testing/user.properties: -------------------------------------------------------------------------------- 1 | # custom setting for jmeter 2 | 3 | # these variables defined in the flow will appear in the results as additional columns 4 | # for some unknown reason, the dockerized version does not accept it using J"user.properties=/var/jmeter/user.properties" 5 | # sample_variables=graphType,operationCount,attributeCount,readCount 6 | 7 | # reports granularity 8 | jmeter.reportgenerator.overall_granularity=1006 9 | --------------------------------------------------------------------------------