├── .gitignore ├── LICENSE ├── README.md ├── benchmarkResult.ipynb ├── build.gradle.kts ├── buildSrc ├── build.gradle.kts ├── settings.gradle.kts └── src │ └── main │ └── kotlin │ └── blogpost │ ├── DownloadGradleProfileTask.kt │ ├── PerformanceTask.kt │ ├── Scenario.kt │ └── TestProject.kt ├── gradle.properties ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat └── settings.gradle.kts /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | .gradle/ 3 | build/ 4 | .kotlin/ 5 | local.properties 6 | /reports -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![JetBrains incubator project](https://jb.gg/badges/incubator-flat-square.svg)](https://confluence.jetbrains.com/display/ALL/JetBrains+on+GitHub) 2 | 3 | ## Introduction 4 | 5 | Use this project to run performance tests on your Gradle project and compare the differences in 6 | performance between [Kotlin K1 and K2 compilers](https://blog.jetbrains.com/kotlin/2023/02/k2-kotlin-2-0/). 7 | 8 | Three test scenarios are covered to benchmark the performance: 9 | 10 | * clean build 11 | * incremental build with non-ABI changes 12 | * incremental build with ABI changes 13 | 14 | After the build finishes, open the [benchmarkResult.ipynb](benchmarkResult.ipynb) Kotlin Notebook to compare the results. 15 | 16 | > [!IMPORTANT] 17 | > You must have the [Kotlin Notebook](https://blog.jetbrains.com/kotlin/2023/07/introducing-kotlin-notebook/) plugin 18 | > installed in IntelliJ IDEA Ultimate to view the results. 19 | 20 | ## Prerequisites 21 | 22 | Ensure beforehand that the project that you want to analyze can be successfully compiled with both Kotlin versions that you want to compare. 23 | By default, this project uses Kotlin 1.9.24 and Kotlin 2.0.0. 24 | 25 | So that the Kotlin version can be automatically configured for your project, in your `build.gradle(.kts)` file, add the `$kotlin_version` parameter: 26 | 27 | ```kotlin 28 | plugins { 29 | id("org.jetbrains.kotlin.jvm") version "$kotlin_version" 30 | } 31 | ``` 32 | 33 | If your project uses a version catalog, update the `settings.gradle.kts` file to override the Kotlin version using the following code: 34 | 35 | ```kotlin 36 | versionCatalogs { 37 | create("libs") { 38 | val kotlinVersion = System.getenv("kotlin_version") 39 | if (kotlinVersion != null) { 40 | version("kotlin", kotlinVersion) 41 | } 42 | } 43 | } 44 | ``` 45 | 46 | > [!IMPORTANT] 47 | > You might encounter new warnings in your builds using Kotlin `2.0.0`. 48 | > If you use the `allWarningsAsErrors` [compiler option](gradle-compiler-options.md#attributes-common-to-jvm-js-and-js-dce), you can remove it to continue without fixing these warnings. 49 | > We recommend that you fix all warnings before proceeding. 50 | 51 | Ensure that the [`JAVA_HOME`](https://docs.oracle.com/cd/E19182-01/821-0917/inst_jdk_javahome_t/index.html) environment variable is set. 52 | If your project involves Android development, make sure that the [`ANDROID_HOME`](https://developer.android.com/tools/variables) 53 | environmental variable is set as well. 54 | 55 | If [dependencies verification](https://docs.gradle.org/8.2.1/userguide/dependency_verification.html#sub:enabling-verification) is enabled, 56 | ensure that the dependencies for both Kotlin `1.9` and Kotlin `2.0` are correctly included in your project setup. 57 | Or [disable/lenient](https://docs.gradle.org/current/userguide/dependency_verification.html#sec:disabling-verification) dependency verification. 58 | 59 | If you are using the `kotlinDsl` plugin in the `buildSrc` subproject, we recommend applying the `kotlin("jvm")` plugin as well. 60 | This prevents issues related to unrecognized output types, such as the new JSON format for build reports available from Kotlin `1.9.23` and Kotlin `2.0.0-RC1`, and ensures compatibility across your project's configuration. 61 | 62 | ## Step 1: Configure project settings 63 | 64 | To configure your project for performance tests, modify your [gradle.properties](gradle.properties) file: 65 | 66 | * Set `project.path` to configure the path to your project. 67 | * Set `project.git.url` and `project.git.commit.sha` to specify the URL and commit of a Git repository from which to clone your project. 68 | * Set `project.git.directory` to specify the directory where the project is stored. 69 | 70 | ## Step 2: Configure build scenarios 71 | 72 | To configure default scenarios for your project, modify your [gradle.properties](gradle.properties) file: 73 | 74 | * Set `scenario.non.abi.changes` to specify files for incremental builds with non-ABI changes. For multiple files, separate them with commas. 75 | * Set `scenario.abi.changes` to specify a single file path for incremental builds with ABI changes. 76 | * Set `scenario.task` to define the default build task; if not set, the `assemble` task will be used by default. 77 | 78 | Gradle profiler allows you to run scenarios with `cli` or `tooling-api`. 79 | Set `scenario.run.using` to specify a way to run scenarios or default `cli` option will be used. 80 | ### Create custom build scenarios 81 | 82 | You can define custom build scenarios for your project by specifying them in your [`build.gradle.kts`](build.gradle.kts) file. 83 | Set up your scenarios using the `PerformanceTask.scenarios` task input or configure a 84 | [Gradle profiler](https://github.com/gradle/gradle-profiler) scenario file using the `PerformanceTask.scenarioFile` task input. 85 | 86 | For example, you can add the following configuration to your [build.gradle.kts](build.gradle.kts) file to use custom scenarios: 87 | 88 | ```kotlin 89 | import org.jetbrains.kotlin.k2.blogpost.PerformanceTask 90 | 91 | // Defines custom scenarios 92 | val customScenarios = listOf(Scenario( 93 | name = "new scenario", 94 | cleanTasks = listOf("clean_task_if_needed"), 95 | kotlinVersion = kotlinVersion, 96 | tasks = listOf("task_to_execute"), 97 | projectDir = project.projectDir, 98 | nonAbiChanges = listOf("path_to_file"), 99 | warmUpRounds = 5, 100 | executionRounds = 5, 101 | )) 102 | 103 | // Registers performance tasks for Kotlin versions 2.0.0 and 1.9.24 104 | val benchmark_2_0 = PerformanceTask.registerPerformanceTask(project, "benchmark_2_0", "2.0.0") { 105 | scenarios.set(customScenarios) 106 | } 107 | 108 | val benchmark_1_9 = PerformanceTask.registerPerformanceTask(project, "benchmark_1_9", "1.9.24") { 109 | scenarios.set(customScenarios) 110 | } 111 | 112 | // Registers a task to run all benchmarks 113 | tasks.register("runBenchmarks") { 114 | dependsOn(benchmark_2_0, benchmark_1_9) 115 | } 116 | ``` 117 | 118 | ## Step 3: Run performance test 119 | 120 | To run performance tests, run the following command in your terminal: 121 | 122 | ```bash 123 | ./gradlew runBenchmarks 124 | ``` 125 | 126 | ## Step 4: (Optional) Visualize results 127 | 128 | > [!IMPORTANT] 129 | > You must have the [Kotlin Notebook](https://blog.jetbrains.com/kotlin/2023/07/introducing-kotlin-notebook/) plugin 130 | > installed in IntelliJ IDEA Ultimate to view the results. 131 | 132 | To analyze the results: 133 | 134 | 1. Open the [benchmarkResult.ipynb](benchmarkResult.ipynb) Kotlin Notebook file. 135 | 2. Run all code cells in the Kotlin Notebook using the `Run All` button to display and compare the produced results. 136 | 137 | ## Troubleshooting 138 | 139 | If you encounter any issues using this project, let us know: 140 | * Join our Kotlin Slack workspace - [get an invite](https://surveys.jetbrains.com/s3/kotlin-slack-sign-up?_gl=1*ju6cbn*_ga*MTA3MTk5NDkzMC4xNjQ2MDY3MDU4*_ga_9J976DJZ68*MTY1ODMzNzA3OS4xMDAuMS4xNjU4MzQwODEwLjYw) and share your experience in the [#k2-early-adopters](https://kotlinlang.slack.com/archives/C03PK0PE257) channel. 141 | * Create an issue in [our issue tracker](https://youtrack.jetbrains.com/newIssue?project=KT&summary=K2+release+migration+issue&description=Describe+the+problem+you+encountered+here.&c=tag+k2-release-migration). -------------------------------------------------------------------------------- /benchmarkResult.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "metadata": {}, 5 | "cell_type": "code", 6 | "source": [ 7 | "val reportFolder = \"reports\"\n", 8 | "val executionRounds = 10" 9 | ], 10 | "outputs": [], 11 | "execution_count": null 12 | }, 13 | { 14 | "metadata": {}, 15 | "cell_type": "code", 16 | "source": [ 17 | "import java.io.File\n", 18 | "import java.util.Properties\n", 19 | "\n", 20 | "val properties = Properties()\n", 21 | "File(\"gradle.properties\").bufferedReader().use {\n", 22 | " properties.load(it)\n", 23 | "}\n", 24 | "\n", 25 | "val executedTask = properties.getProperty(\"scenario.task\") ?: \"assemble\"\n", 26 | "\n", 27 | "val projectPath = properties.getProperty(\"project.path\") ?: properties.getProperty(\"project.git.directory\")\n", 28 | "\n", 29 | "val projectName: String = projectPath?.substringAfterLast(\"\\\\\")?.substringAfterLast(\"/\")?.removeSuffix(\".git\") ?: \"User\"" 30 | ], 31 | "outputs": [], 32 | "execution_count": null 33 | }, 34 | { 35 | "metadata": {}, 36 | "cell_type": "code", 37 | "source": [ 38 | "enum class CompilerMetrics(val readableName: String) {\n", 39 | " GRADLE_TASK(\"Total Gradle task time\"),\n", 40 | "\n", 41 | " COMPILATION_ROUND(\"Sources compilation round\"),\n", 42 | " \n", 43 | " //compiler metrics\n", 44 | " COMPILER_INITIALIZATION(\"Compiler initialization time\"),\n", 45 | " CODE_ANALYSIS(\"Compiler code analysis\"),\n", 46 | " IR_TRANSLATION(\"Compiler IR translation\"),\n", 47 | " IR_LOWERING(\"Compiler IR lowering\"),\n", 48 | " IR_GENERATION(\"Compiler IR generation\"),\n", 49 | " ;\n", 50 | "}" 51 | ], 52 | "outputs": [], 53 | "execution_count": null 54 | }, 55 | { 56 | "metadata": {}, 57 | "cell_type": "code", 58 | "source": "%use dataframe", 59 | "outputs": [], 60 | "execution_count": null 61 | }, 62 | { 63 | "metadata": {}, 64 | "cell_type": "code", 65 | "source": [ 66 | "import java.io.File\n", 67 | "import org.jetbrains.kotlinx.dataframe.DataFrame\n", 68 | "import java.util.logging.Logger\n", 69 | "import java.util.logging.Level\n", 70 | "\n", 71 | "\n", 72 | "fun validateAndReadJsonReportFile(file: File, logger: Logger): DataFrame<*>? {\n", 73 | " if (file.extension != \"json\" || file.readText().isBlank()) {\n", 74 | " logger.log(Level.FINE, \"${file.path} file will be ignored\")\n", 75 | " return null\n", 76 | " }\n", 77 | " val report = DataFrame.read(file)\n", 78 | "\n", 79 | " if (!(report.get(\"startParameters\").get(\"tasks\")[0] as List).contains(executedTask)) {\n", 80 | " logger.log(Level.FINE, \"${file.path} file will be ignored because the reports does not contains $executedTask\")\n", 81 | " return null\n", 82 | " }\n", 83 | " return report\n", 84 | " \n", 85 | "}" 86 | ], 87 | "outputs": [], 88 | "execution_count": null 89 | }, 90 | { 91 | "metadata": {}, 92 | "cell_type": "code", 93 | "source": [ 94 | "\n", 95 | "import org.jetbrains.kotlinx.dataframe.DataFrame\n", 96 | "import java.util.logging.Logger\n", 97 | "\n", 98 | "//Json report print time in milliseconds \n", 99 | "fun readBuildMetricsFromJsonReportFile(file: File, logger: Logger): Map? {\n", 100 | " val report = validateAndReadJsonReportFile(file, logger)\n", 101 | " \n", 102 | " if (report == null) {\n", 103 | " logger.info(\"${file.path} will be skipped\")\n", 104 | " return null\n", 105 | " }\n", 106 | " \n", 107 | " val timeMetrics = report.get(\"aggregatedMetrics\").get(\"buildTimes\").get(\"buildTimesNs\")[0] as DataRow<*>\n", 108 | "\n", 109 | " return CompilerMetrics.values().associate {\n", 110 | " val value = timeMetrics.getOrNull(it.name)?.let {\n", 111 | " when (it) {\n", 112 | " is Int -> it.toLong()\n", 113 | " is Long -> it\n", 114 | " is String -> it.toLong()\n", 115 | " else -> throw IllegalStateException(\"Unknown type ${it.javaClass} to convert to Long\")\n", 116 | " }\n", 117 | " }\n", 118 | " Pair(it, value)\n", 119 | " }\n", 120 | "}" 121 | ], 122 | "outputs": [], 123 | "execution_count": null 124 | }, 125 | { 126 | "metadata": {}, 127 | "cell_type": "code", 128 | "source": [ 129 | "import kotlin.streams.toList\n", 130 | "import java.util.logging.Logger\n", 131 | "\n", 132 | "\n", 133 | "//Text report print time in seconds with 2 decemal digits, so there is an error during conversion to nanoSeconds\n", 134 | "fun readTextReportFile(file: File, logger: Logger): Map? {\n", 135 | " val errorMessge = when {\n", 136 | " file.extension != \"txt\" -> \"Only txt reports are going to be processed\"\n", 137 | " file.readText().isBlank() -> \"Blank file will be skipped\"\n", 138 | " !file.readText().contains(\"tasks = [$executedTask]\") -> \"Report file does not contains `$executedTask` task execution\"\n", 139 | " else -> null\n", 140 | " }\n", 141 | " if (errorMessge != null) {\n", 142 | " logger.info(errorMessge)\n", 143 | " return null\n", 144 | " }\n", 145 | " \n", 146 | " val aggregatedTimeMetrics = file.bufferedReader().use {\n", 147 | " it.lines().dropWhile { !it.startsWith(\"Time metrics:\") }.takeWhile { it.isNotBlank() }.toList()\n", 148 | " }\n", 149 | "\n", 150 | " return CompilerMetrics.values().associate { metric ->\n", 151 | " val metricLine = aggregatedTimeMetrics.firstOrNull { it.trim().startsWith(metric.readableName) }\n", 152 | " val value = metricLine?.substringAfter(\": \")?.substringBefore(\" \")?.replace(\",\", \"\")?.toDouble()\n", 153 | " Pair(metric, value?.times(1_000_000_000)?.toLong())\n", 154 | " }\n", 155 | "}\n" 156 | ], 157 | "outputs": [], 158 | "execution_count": null 159 | }, 160 | { 161 | "metadata": {}, 162 | "cell_type": "markdown", 163 | "source": [ 164 | "# Read compiler metrics\n" 165 | ] 166 | }, 167 | { 168 | "metadata": {}, 169 | "cell_type": "code", 170 | "source": "%use kandy", 171 | "outputs": [], 172 | "execution_count": null 173 | }, 174 | { 175 | "metadata": {}, 176 | "cell_type": "code", 177 | "source": [ 178 | "import org.jetbrains.kotlinx.dataframe.api.toDataFrame\n", 179 | "import java.util.logging.Logger\n", 180 | "import java.nio.file.Files\n", 181 | "import kotlin.io.path.Path\n", 182 | "\n", 183 | "fun readMetrics(\n", 184 | " directory: File,\n", 185 | " logger: Logger,\n", 186 | " readReportFile: (File, Logger) -> Map?\n", 187 | ") = directory.listFiles()\n", 188 | " ?.sortedByDescending(File::lastModified)\n", 189 | " ?.mapNotNull { readReportFile(it, logger) }\n", 190 | " ?.take(executionRounds)\n", 191 | " ?.toList() ?: emptyList>().also { println(\"$directory scenario directory is empty\") }\n", 192 | "\n", 193 | "fun collectMeanValue(\n", 194 | " metrics: List>,\n", 195 | "): Map {\n", 196 | " val rows = CompilerMetrics.values().map { compilerMetric ->\n", 197 | " val values = metrics.map { it[compilerMetric] }.filterNotNull()\n", 198 | " Pair(compilerMetric.name, values)\n", 199 | " }.filter { \n", 200 | " val allValuesPresent = (it.second.size == executionRounds)\n", 201 | " if (!allValuesPresent) {\n", 202 | " println(\"Unable to calculate mean value for ${it.first}, only ${it.second.size} runs are found\")\n", 203 | " }\n", 204 | " allValuesPresent\n", 205 | " }.toTypedArray()\n", 206 | "\n", 207 | " val meanValue = dataFrameOf(*rows).mean()\n", 208 | "\n", 209 | " return CompilerMetrics.values().associate {\n", 210 | " Pair(it.name, meanValue.getOrNull(it.name))\n", 211 | " }\n", 212 | "} \n" 213 | ], 214 | "outputs": [], 215 | "execution_count": null 216 | }, 217 | { 218 | "metadata": {}, 219 | "cell_type": "markdown", 220 | "source": "# Collect data into dataframe" 221 | }, 222 | { 223 | "metadata": {}, 224 | "cell_type": "code", 225 | "source": "", 226 | "outputs": [], 227 | "execution_count": null 228 | }, 229 | { 230 | "metadata": {}, 231 | "cell_type": "code", 232 | "source": [ 233 | "import java.util.logging.Logger\n", 234 | "\n", 235 | "val kotlinVersionColumn = \"kotlinVersion\"\n", 236 | "val buildScenarioColumn = \"buildScenario\"\n", 237 | "\n", 238 | "val columnNames = CompilerMetrics.values().map { it.name }.toMutableList()\n", 239 | "columnNames.add(kotlinVersionColumn)\n", 240 | "columnNames.add(buildScenarioColumn)\n", 241 | "\n", 242 | "fun collectResultsIntoDataFrame(\n", 243 | " directory: File,\n", 244 | " logger: Logger,\n", 245 | " readReportFile: (File, Logger) -> Map?,\n", 246 | "): AnyFrame {\n", 247 | " val rows = directory.listFiles()?.filter { it.isDirectory() }?.flatMap { versionDir ->\n", 248 | " versionDir.listFiles()?.filter { it.isDirectory() }?.map { scenarioDir ->\n", 249 | " val buildMetrics = readMetrics(scenarioDir, logger, readReportFile)\n", 250 | " //rename scenarios for graphics only\n", 251 | " val scenario = when (scenarioDir.name) {\n", 252 | " \"clean_build\" -> \"Clean build\"\n", 253 | " \"incremental_abi_build\" -> \"Incremental build for ABI changes\"\n", 254 | " \"incremental_non_abi_build\" -> \"Incremental build for non-ABI changes\"\n", 255 | " else -> scenarioDir.name\n", 256 | " }\n", 257 | "\n", 258 | " logger.info(\"\\n Validate \\'$scenario\\' scenario for ${versionDir.name} kotlin version\")\n", 259 | " val row = collectMeanValue(buildMetrics).toMutableMap()\n", 260 | " row.put(kotlinVersionColumn, versionDir.name)\n", 261 | " row.put(buildScenarioColumn, scenario)\n", 262 | " row\n", 263 | " } ?: emptyList>().also { println(\"'$versionDir' version directory is empty\") }\n", 264 | " } ?: emptyList>().also{ println(\"'$directory' report directory is empty\") }\n", 265 | "\n", 266 | " val db = columnNames.map { compilerMetric -> Pair(compilerMetric, rows.map { it[compilerMetric] })}.toTypedArray()\n", 267 | " return dataFrameOf(*db)\n", 268 | "}" 269 | ], 270 | "outputs": [], 271 | "execution_count": null 272 | }, 273 | { 274 | "metadata": {}, 275 | "cell_type": "code", 276 | "source": [ 277 | "import java.util.concurrent.TimeUnit\n", 278 | "import kotlin.time.DurationUnit\n", 279 | "import kotlin.time.toDuration\n", 280 | "\n", 281 | "enum class TimeTransformation(val printTime: String, val transformation: (Double?) -> Double?) {\n", 282 | " NANOSECONDS(\"nanosaconds\", { it }),\n", 283 | " MILLISECONDS(\"milliseconds\", { \n", 284 | " it?.toDuration(DurationUnit.NANOSECONDS)?.toDouble(DurationUnit.MILLISECONDS) \n", 285 | " }),\n", 286 | " SECONDS(\"seconds\", {\n", 287 | " it?.toDuration(DurationUnit.NANOSECONDS)?.toDouble(DurationUnit.SECONDS) \n", 288 | " })\n", 289 | " ;\n", 290 | "}\n", 291 | "\n", 292 | "fun getTimeTranformationForColumn(df: DataFrame, column: String): TimeTransformation {\n", 293 | " val meanValue = df.mean(column, skipNA = true)\n", 294 | " val ratio = 1_000\n", 295 | " return when {\n", 296 | " meanValue < 10 * ratio -> TimeTransformation.NANOSECONDS\n", 297 | " meanValue < 10 * ratio * ratio -> TimeTransformation.MILLISECONDS\n", 298 | " else -> TimeTransformation.SECONDS\n", 299 | " }\n", 300 | "}" 301 | ], 302 | "outputs": [], 303 | "execution_count": null 304 | }, 305 | { 306 | "metadata": {}, 307 | "cell_type": "code", 308 | "source": [ 309 | "import java.util.logging.FileHandler\n", 310 | "import java.util.logging.Logger\n", 311 | "\n", 312 | "val logger = Logger.getLogger(\"benchmarks\")\n", 313 | "val fileHandler = FileHandler(\"benchmarks.log\", false)\n", 314 | "logger.addHandler(fileHandler)\n", 315 | "\n", 316 | "\n", 317 | "var df = collectResultsIntoDataFrame(File(reportFolder), logger, ::readBuildMetricsFromJsonReportFile)\n", 318 | " .sortBy(kotlinVersionColumn)\n", 319 | "val timeTransformation = getTimeTranformationForColumn(df, CompilerMetrics.COMPILATION_ROUND.name)\n", 320 | "\n", 321 | " CompilerMetrics.values().forEach { \n", 322 | " df = df.update(it.name).with { timeTransformation.transformation(it as Double) }\n", 323 | " }\n", 324 | "\n", 325 | "val dataFrame = df\n", 326 | "val scenarios = dataFrame[buildScenarioColumn]" 327 | ], 328 | "outputs": [], 329 | "execution_count": null 330 | }, 331 | { 332 | "metadata": {}, 333 | "cell_type": "code", 334 | "source": [ 335 | "val plots = scenarios.distinct().toList().map { scenario ->\n", 336 | " dataFrame.filter { buildScenario == scenario }.plot {\n", 337 | " layout {\n", 338 | " title = \"$projectName project\"\n", 339 | " subtitle = \"$scenario\"\n", 340 | " style { \n", 341 | " legend {\n", 342 | " position = LegendPosition.Top\n", 343 | " }\n", 344 | " }\n", 345 | " }\n", 346 | " bars {\n", 347 | " x(kotlinVersionColumn) {\n", 348 | " axis.name = \"Gradle compilation time\"\n", 349 | " }\n", 350 | " y(CompilerMetrics.COMPILATION_ROUND.name) {\n", 351 | " axis.name = \"Time (${timeTransformation.printTime})\"\n", 352 | " }\n", 353 | " }\n", 354 | " \n", 355 | " }\n", 356 | "}\n", 357 | "plotGrid(plots, 1)" 358 | ], 359 | "outputs": [], 360 | "execution_count": null 361 | }, 362 | { 363 | "metadata": {}, 364 | "cell_type": "code", 365 | "source": "val incrementalScenatiosLabels = listOf(Pair(\"Incremental build for ABI changes\", \"ABI changes\"), Pair(\"Incremental build for non-ABI changes\", \"non-ABI changes\"))", 366 | "outputs": [], 367 | "execution_count": null 368 | }, 369 | { 370 | "metadata": {}, 371 | "cell_type": "code", 372 | "source": [ 373 | "fun gradleBuildTime(scenarios: List, name: String, scenarioBreaksLabeled: List>? = null) =\n", 374 | " dataFrame.filter { buildScenario in scenarios }\n", 375 | " .plot {\n", 376 | " layout {\n", 377 | " title = \"$projectName project - $name\"\n", 378 | " style {\n", 379 | " legend {\n", 380 | " position = LegendPosition.Top\n", 381 | " }\n", 382 | " }\n", 383 | " }\n", 384 | " bars {\n", 385 | " x(buildScenarioColumn) {\n", 386 | " axis.name = \"Gradle build time\"\n", 387 | " scenarioBreaksLabeled?.let {\n", 388 | " axis.breaksLabeled(it.map { it.first }, it.map { it.second })\n", 389 | " }\n", 390 | " }\n", 391 | " y(GRADLE_TASK.name) {\n", 392 | " axis.name = \"Time (${timeTransformation.printTime})\"\n", 393 | " }\n", 394 | " fillColor(kotlinVersionColumn) {\n", 395 | " legend.name = \"\"\n", 396 | " }\n", 397 | " }\n", 398 | " }" 399 | ], 400 | "outputs": [], 401 | "execution_count": null 402 | }, 403 | { 404 | "metadata": {}, 405 | "cell_type": "code", 406 | "source": "gradleBuildTime(listOf(\"Clean build\"), \"Clean build scenario\")", 407 | "outputs": [], 408 | "execution_count": null 409 | }, 410 | { 411 | "metadata": {}, 412 | "cell_type": "code", 413 | "source": "gradleBuildTime(listOf(\"Incremental build for ABI changes\", \"Incremental build for non-ABI changes\"), \"Incremental build scenarios\", incrementalScenatiosLabels)\n", 414 | "outputs": [], 415 | "execution_count": null 416 | }, 417 | { 418 | "metadata": {}, 419 | "cell_type": "code", 420 | "source": [ 421 | "fun kotlinCompilationTime(\n", 422 | " scenarios: List,\n", 423 | " name: String,\n", 424 | " scenarioBreaksLabeled: List>? = null\n", 425 | ") = dataFrame.filter { buildScenario in scenarios }\n", 426 | " .plot {\n", 427 | " layout {\n", 428 | " title = \"$projectName project - $name\"\n", 429 | " style {\n", 430 | " legend {\n", 431 | " position = LegendPosition.Top\n", 432 | " }\n", 433 | " }\n", 434 | " }\n", 435 | " bars {\n", 436 | " x(buildScenarioColumn) {\n", 437 | " axis.name = \"Kotlin compilation time\"\n", 438 | " scenarioBreaksLabeled?.let {\n", 439 | " axis.breaksLabeled(it.map { it.first }, it.map { it.second })\n", 440 | " }\n", 441 | " }\n", 442 | " y(COMPILATION_ROUND.name) {\n", 443 | " axis.name = \"Time (${timeTransformation.printTime})\"\n", 444 | " }\n", 445 | " fillColor(kotlinVersionColumn) {\n", 446 | " legend.name = \"\"\n", 447 | " }\n", 448 | " }\n", 449 | " }\n", 450 | " \n" 451 | ], 452 | "outputs": [], 453 | "execution_count": null 454 | }, 455 | { 456 | "metadata": {}, 457 | "cell_type": "code", 458 | "source": "kotlinCompilationTime(listOf(\"Clean build\"), \"Clean build scenario\")\n", 459 | "outputs": [], 460 | "execution_count": null 461 | }, 462 | { 463 | "metadata": {}, 464 | "cell_type": "code", 465 | "source": "kotlinCompilationTime(listOf(\"Incremental build for ABI changes\", \"Incremental build for non-ABI changes\"), \"Incremental build scenarios\", incrementalScenatiosLabels)", 466 | "outputs": [], 467 | "execution_count": null 468 | }, 469 | { 470 | "metadata": {}, 471 | "cell_type": "markdown", 472 | "source": "# Compare Gradle and Compiler time" 473 | }, 474 | { 475 | "metadata": {}, 476 | "cell_type": "code", 477 | "source": [ 478 | "import org.jetbrains.kotlinx.kandy.ir.scale.PositionalContinuousScale\n", 479 | "import org.jetbrains.kotlinx.kandy.ir.scale.PositionalScale\n", 480 | "import org.jetbrains.kotlinx.kandy.ir.scale.PositionalTransform\n", 481 | "\n", 482 | "fun getPlotForCompilerMetrics(\n", 483 | " scenario: String,\n", 484 | " graphicsCaption: String,\n", 485 | " vararg metrics: Pair\n", 486 | ") = dataFrame\n", 487 | " .gather(*metrics.map { it.first.name }.toTypedArray()).into(\"metric\", \"time_ns\")\n", 488 | " .filter { it[\"buildScenario\"] == scenario }\n", 489 | " .plot {\n", 490 | " layout {\n", 491 | " title = \"$projectName project\"\n", 492 | " subtitle = scenario\n", 493 | " caption = graphicsCaption\n", 494 | " style {\n", 495 | " legend {\n", 496 | " position = LegendPosition.Top\n", 497 | " }\n", 498 | " }\n", 499 | " size = Pair(800, 300)\n", 500 | " }\n", 501 | " barsH {\n", 502 | " y(\"kotlinVersion\"()) {\n", 503 | " axis.name = \"Kotlin version\"\n", 504 | " }\n", 505 | " x(\"time_ns\"()) {\n", 506 | " axis.name = \"Time (${timeTransformation.printTime})\"\n", 507 | " }\n", 508 | " fillColor(\"metric\"()) {\n", 509 | " legend {\n", 510 | " name = \"\"\n", 511 | " breaksLabeled(*metrics.map { Pair(it.first.name, it.second) }.toTypedArray())\n", 512 | " }\n", 513 | " }\n", 514 | " position = Position.stack()\n", 515 | "\n", 516 | " }\n", 517 | " }" 518 | ], 519 | "outputs": [], 520 | "execution_count": null 521 | }, 522 | { 523 | "metadata": {}, 524 | "cell_type": "code", 525 | "source": [ 526 | "val versionByScenario =\n", 527 | " scenarios.distinct().toList().associate { name -> Pair(name as String, getPlotForCompilerMetrics(name, \"Gradle compilation breakdown\", CompilerMetrics.COMPILATION_ROUND to \"Kotlin Compiler time\", \n", 528 | " CompilerMetrics.GRADLE_TASK to \"Gradle time\")) }" 529 | ], 530 | "outputs": [], 531 | "execution_count": null 532 | }, 533 | { 534 | "metadata": {}, 535 | "cell_type": "code", 536 | "source": "versionByScenario.get(\"Clean build\")", 537 | "outputs": [], 538 | "execution_count": null 539 | }, 540 | { 541 | "metadata": {}, 542 | "cell_type": "code", 543 | "source": "versionByScenario.get(\"Incremental build for non-ABI changes\")", 544 | "outputs": [], 545 | "execution_count": null 546 | }, 547 | { 548 | "metadata": {}, 549 | "cell_type": "code", 550 | "source": "versionByScenario.get(\"Incremental build for ABI changes\")", 551 | "outputs": [], 552 | "execution_count": null 553 | }, 554 | { 555 | "metadata": {}, 556 | "cell_type": "markdown", 557 | "source": "# Collect Compiler metrics" 558 | }, 559 | { 560 | "metadata": {}, 561 | "cell_type": "code", 562 | "source": [ 563 | "val compilerMetrics = scenarios.distinct().toList().associate {\n", 564 | " Pair(\n", 565 | " it as String, getPlotForCompilerMetrics(\n", 566 | " it, \"Kotlin compilation breakdown\",\n", 567 | " CompilerMetrics.COMPILER_INITIALIZATION to \"Initialization phase\",\n", 568 | " CompilerMetrics.CODE_ANALYSIS to \"Analysis phase\",\n", 569 | " CompilerMetrics.IR_TRANSLATION to \"Translation phase\",\n", 570 | " CompilerMetrics.IR_LOWERING to \"Lowering\",\n", 571 | " CompilerMetrics.IR_GENERATION to \"Generation\"\n", 572 | " )\n", 573 | " )\n", 574 | "}" 575 | ], 576 | "outputs": [], 577 | "execution_count": null 578 | }, 579 | { 580 | "metadata": {}, 581 | "cell_type": "code", 582 | "source": "compilerMetrics.get(\"Clean build\") \n", 583 | "outputs": [], 584 | "execution_count": null 585 | }, 586 | { 587 | "metadata": {}, 588 | "cell_type": "code", 589 | "source": "compilerMetrics.get(\"Incremental build for non-ABI changes\") ", 590 | "outputs": [], 591 | "execution_count": null 592 | }, 593 | { 594 | "metadata": {}, 595 | "cell_type": "code", 596 | "source": "compilerMetrics.get(\"Incremental build for ABI changes\") ", 597 | "outputs": [], 598 | "execution_count": null 599 | }, 600 | { 601 | "metadata": {}, 602 | "cell_type": "markdown", 603 | "source": "# Analyze executed tasks" 604 | }, 605 | { 606 | "metadata": {}, 607 | "cell_type": "code", 608 | "source": [ 609 | "import java.util.logging.Logger\n", 610 | "\n", 611 | "fun readTasksJsonReportFile(file: File, logger: Logger): Map? {\n", 612 | " val report = validateAndReadJsonReportFile(file, logger)\n", 613 | " if (report == null) {\n", 614 | " logger.fine(\"${file.path} file will be skipped\")\n", 615 | " return null\n", 616 | " }\n", 617 | "\n", 618 | " val taskExecution = HashMap()\n", 619 | "\n", 620 | " logger.info(\"${file.path} will be avalyzed\")\n", 621 | "\n", 622 | " val buildOperationRecord = report.get(\"buildOperationRecord\")[0] as DataFrame<*>\n", 623 | "\n", 624 | " buildOperationRecord.forEach {\n", 625 | " val name = it.getValue(\"path\").substringAfterLast(\":\")\n", 626 | "\n", 627 | " val time = it.getValue>(\"buildMetrics\")\n", 628 | " .getValue>(\"buildTimes\")\n", 629 | " .getValue>(\"buildTimesNs\")\n", 630 | " .get(CompilerMetrics.GRADLE_TASK.name)\n", 631 | "\n", 632 | "\n", 633 | " val lonValue: Long? = when (time) {\n", 634 | " is Int -> time.toLong()\n", 635 | " is Long -> time\n", 636 | " is String -> time.toLong()\n", 637 | " is Number -> time.toLong()\n", 638 | " else -> {\n", 639 | " logger.warning(\"Unknow type garadle task execution time: $time\")\n", 640 | " null\n", 641 | " }\n", 642 | " }\n", 643 | "\n", 644 | " lonValue?.also { taskExecution[name] = taskExecution.getOrDefault(name, 0) + it }\n", 645 | " }\n", 646 | "\n", 647 | " return taskExecution\n", 648 | "}" 649 | ], 650 | "outputs": [], 651 | "execution_count": null 652 | }, 653 | { 654 | "metadata": {}, 655 | "cell_type": "code", 656 | "source": [ 657 | "fun collectMapsToDataframe(map: List>): DataFrame<*> {\n", 658 | " val columns = map.flatMap { it.keys }.distinct()\n", 659 | "\n", 660 | " val rows = map.flatMap { row ->\n", 661 | " columns.map { row[it] }\n", 662 | " }\n", 663 | "\n", 664 | " return dataFrameOf(columns, rows)\n", 665 | "}\n" 666 | ], 667 | "outputs": [], 668 | "execution_count": null 669 | }, 670 | { 671 | "metadata": {}, 672 | "cell_type": "code", 673 | "source": [ 674 | "fun collectTaskExecutionResults(\n", 675 | " directory: File,\n", 676 | " logger: Logger\n", 677 | "): List> {\n", 678 | " logger.info(\"read ${directory.absolutePath} directory\")\n", 679 | " return directory.listFiles()?.filter { it.isDirectory() }?.flatMap { versionDir ->\n", 680 | " logger.info(\"read reports for '${versionDir.name}' version\")\n", 681 | " versionDir.listFiles()?.filter { it.isDirectory() }?.mapNotNull { scenarioDir ->\n", 682 | " logger.info(\"read reports for '${scenarioDir.name}' scenario\")\n", 683 | " val executedTasksTimes = scenarioDir.listFiles()?.filter { it.isFile() }\n", 684 | " ?.sortedByDescending { it.lastModified() }\n", 685 | " ?.mapNotNull { file ->\n", 686 | " readTasksJsonReportFile(file, logger).also {\n", 687 | " logger.info(\"found tasks for '${scenarioDir.name}' scenarion and ${versionDir.name} version: $it\")\n", 688 | " }\n", 689 | " }\n", 690 | " ?.take(executionRounds)\n", 691 | " \n", 692 | " if (executedTasksTimes == null) {\n", 693 | " logger.warning(\"No build reports found for '${scenarioDir.name}' scenario and '${versionDir.name}' version\")\n", 694 | " null\n", 695 | " } else {\n", 696 | " \n", 697 | " val meanRow = collectMapsToDataframe(executedTasksTimes).mean(skipNA = true).toMap().toMutableMap()\n", 698 | " meanRow[kotlinVersionColumn] = versionDir.name\n", 699 | " meanRow[buildScenarioColumn] = scenarioDir.name\n", 700 | " meanRow.toMap()\n", 701 | " }\n", 702 | " } ?: emptyList() \n", 703 | " } ?: emptyList()\n", 704 | "}" 705 | ], 706 | "outputs": [], 707 | "execution_count": null 708 | }, 709 | { 710 | "metadata": {}, 711 | "cell_type": "code", 712 | "source": [ 713 | "\n", 714 | "val logger = Logger.getLogger(\"task_result\")\n", 715 | "logger.addHandler(fileHandler)\n", 716 | "\n", 717 | "val taskExecutions = collectTaskExecutionResults(File(reportFolder), logger)\n", 718 | "val taskExecutionsDataFrame = collectMapsToDataframe(taskExecutions) \n", 719 | "val tasks = taskExecutionsDataFrame.columns().map { it.name() }.filter { it != buildScenarioColumn && it != kotlinVersionColumn }.toTypedArray()\n", 720 | "\n", 721 | "fun getPlotForTasks(\n", 722 | " scenario: String,\n", 723 | ") = taskExecutionsDataFrame\n", 724 | " .gather(*tasks).into(\"task\", \"time_ns\")\n", 725 | " .filter { it[buildScenarioColumn] == scenario }\n", 726 | " .sortBy(\"time_ns\")\n", 727 | " .plot {\n", 728 | " layout {\n", 729 | " title = \"$projectName project\"\n", 730 | " subtitle = scenario\n", 731 | " style {\n", 732 | " legend {\n", 733 | " position = LegendPosition.Top\n", 734 | " }\n", 735 | " }\n", 736 | " size = Pair(800, 300)\n", 737 | " }\n", 738 | " barsH {\n", 739 | " y(\"kotlinVersion\"()) {\n", 740 | " axis.name = \"Kotlin version\"\n", 741 | " }\n", 742 | " x(\"time_ns\"()) \n", 743 | " fillColor(\"task\"()) \n", 744 | " position = Position.stack()\n", 745 | " }\n", 746 | " }\n", 747 | "\n" 748 | ], 749 | "outputs": [], 750 | "execution_count": null 751 | }, 752 | { 753 | "metadata": {}, 754 | "cell_type": "code", 755 | "source": "getPlotForTasks(\"clean_build\")", 756 | "outputs": [], 757 | "execution_count": null 758 | }, 759 | { 760 | "metadata": {}, 761 | "cell_type": "code", 762 | "source": "", 763 | "outputs": [], 764 | "execution_count": null 765 | } 766 | ], 767 | "metadata": { 768 | "kernelspec": { 769 | "display_name": "Kotlin", 770 | "language": "kotlin", 771 | "name": "kotlin" 772 | }, 773 | "language_info": { 774 | "name": "kotlin", 775 | "version": "1.9.0", 776 | "mimetype": "text/x-kotlin", 777 | "file_extension": ".kt", 778 | "pygments_lexer": "kotlin", 779 | "codemirror_mode": "text/x-kotlin", 780 | "nbconvert_exporter": "" 781 | }, 782 | "ktnbPluginMetadata": { 783 | "projectDependencies": [ 784 | "buildSrc" 785 | ] 786 | } 787 | }, 788 | "nbformat": 4, 789 | "nbformat_minor": 0 790 | } 791 | -------------------------------------------------------------------------------- /build.gradle.kts: -------------------------------------------------------------------------------- 1 | import org.jetbrains.kotlin.k2.blogpost.DownloadGradleProfileTask 2 | import org.jetbrains.kotlin.k2.blogpost.PerformanceTask 3 | 4 | plugins { 5 | id("org.jetbrains.kotlin.jvm") version "1.9.24" 6 | `kotlin-dsl` 7 | } 8 | 9 | repositories { 10 | mavenCentral() 11 | } 12 | 13 | val downloadGradleProfileTask = DownloadGradleProfileTask.registerTask(project) 14 | 15 | val cleanGeneratedReports_2_0 = tasks.register("cleanGeneratedReports2_0") { 16 | delete(layout.projectDirectory.dir("reports/2.0.0")) 17 | } 18 | 19 | val benchmark_2_0 = PerformanceTask.registerPerformanceTask(project, "benchmark_2_0", "2.0.0") { 20 | gradleProfilerDir.set(downloadGradleProfileTask.flatMap { it.gradleProfilerDir }) 21 | dependsOn(cleanGeneratedReports_2_0) 22 | dependsOn(downloadGradleProfileTask) 23 | } 24 | 25 | val cleanGeneratedReports_1_9 = tasks.register("cleanGeneratedReports1_9") { 26 | delete(layout.projectDirectory.dir("reports/1.9.24")) 27 | } 28 | 29 | val benchmark_1_9 = PerformanceTask.registerPerformanceTask(project, "benchmark_1_9", "1.9.24") { 30 | gradleProfilerDir.set(downloadGradleProfileTask.flatMap { it.gradleProfilerDir }) 31 | dependsOn(cleanGeneratedReports_1_9) 32 | dependsOn(downloadGradleProfileTask) 33 | } 34 | 35 | tasks.register("runBenchmarks") { 36 | group = "benchmarks" 37 | description = "Run all benchmarks" 38 | 39 | dependsOn(benchmark_2_0, benchmark_1_9) 40 | } 41 | -------------------------------------------------------------------------------- /buildSrc/build.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | id("org.jetbrains.kotlin.jvm") version "1.9.24" 3 | `kotlin-dsl` 4 | } 5 | 6 | dependencies { 7 | implementation("org.eclipse.jgit:org.eclipse.jgit:6.9.0.202403050737-r") 8 | implementation("com.squareup.okhttp3:okhttp:4.12.0") 9 | } 10 | 11 | repositories { 12 | mavenLocal() 13 | mavenCentral() 14 | } -------------------------------------------------------------------------------- /buildSrc/settings.gradle.kts: -------------------------------------------------------------------------------- 1 | pluginManagement { 2 | repositories { 3 | google() 4 | mavenCentral() 5 | gradlePluginPortal() 6 | } 7 | } -------------------------------------------------------------------------------- /buildSrc/src/main/kotlin/blogpost/DownloadGradleProfileTask.kt: -------------------------------------------------------------------------------- 1 | package org.jetbrains.kotlin.k2.blogpost 2 | 3 | import okhttp3.OkHttpClient 4 | import okhttp3.Request 5 | import org.gradle.api.DefaultTask 6 | import org.gradle.api.Project 7 | import org.gradle.api.file.Directory 8 | import org.gradle.api.file.ProjectLayout 9 | import org.gradle.api.provider.Provider 10 | import org.gradle.api.tasks.OutputDirectory 11 | import java.io.File 12 | import java.io.IOException 13 | import java.util.zip.ZipInputStream 14 | import javax.inject.Inject 15 | import org.gradle.api.file.RegularFile 16 | import org.gradle.api.tasks.TaskAction 17 | import org.gradle.api.tasks.TaskProvider 18 | 19 | abstract class DownloadGradleProfileTask @Inject constructor( 20 | projectLayout: ProjectLayout, 21 | ) : DefaultTask() { 22 | 23 | @get:OutputDirectory 24 | val gradleProfilerDir: Provider = projectLayout.buildDirectory.dir("gradle-profiler-$GRADLE_PROFILER_VERSION") 25 | 26 | private val gradleProfilerBin: Provider = gradleProfilerDir.map { 27 | it.dir("bin") 28 | .run { 29 | if (System.getProperty("os.name").contains("windows", ignoreCase = true)) { 30 | file("gradle-profiler.bat") 31 | } else { 32 | file("gradle-profiler") 33 | } 34 | } 35 | } 36 | 37 | companion object { 38 | private const val GRADLE_PROFILER_VERSION = "0.21.0-alpha-4" 39 | private const val GRADLE_PROFILER_URL: String = 40 | "https://repo1.maven.org/maven2/org/gradle/profiler/gradle-profiler/$GRADLE_PROFILER_VERSION/gradle-profiler-$GRADLE_PROFILER_VERSION.zip" 41 | 42 | private val String.dropLeadingDir: String get() = substringAfter('/') 43 | private const val DOWNLOAD_GRADLE_TASK_NAME = "downloadGradleProfile" 44 | 45 | fun registerTask(project: Project) = 46 | project.tasks.register( 47 | DOWNLOAD_GRADLE_TASK_NAME, 48 | DownloadGradleProfileTask::class.java 49 | ) { 50 | description = "Download gradle profiler" 51 | group = "benchmarks" 52 | } 53 | } 54 | 55 | 56 | @TaskAction 57 | fun run() { 58 | downloadGradleProfilerIfNotAvailable(gradleProfilerDir.get().asFile) 59 | } 60 | 61 | private fun downloadGradleProfilerIfNotAvailable(gradleProfilerDir: File) { 62 | if (gradleProfilerDir.listFiles().isNullOrEmpty()) { 63 | downloadAndExtractGradleProfiler(gradleProfilerDir) 64 | } else { 65 | logger.info("Gradle profiler has been already downloaded") 66 | } 67 | } 68 | 69 | private fun downloadAndExtractGradleProfiler(gradleProfilerDir: File) { 70 | logger.info("Downloading gradle-profiler into ${gradleProfilerDir.absolutePath}") 71 | 72 | gradleProfilerDir.mkdirs() 73 | 74 | val okHttpClient = OkHttpClient() 75 | val request = Request.Builder() 76 | .get() 77 | .url(GRADLE_PROFILER_URL) 78 | .build() 79 | val response = okHttpClient.newCall(request).execute() 80 | if (!response.isSuccessful) { 81 | throw IOException("Failed to download gradle-profiler, error code: ${response.code}") 82 | } 83 | 84 | val contentLength = response.body!!.contentLength() 85 | var downloadedLength = 0L 86 | logger.debug("Downloading: ") 87 | response.body!!.byteStream().buffered().use { responseContent -> 88 | ZipInputStream(responseContent).use { zip -> 89 | var zipEntry = zip.nextEntry 90 | while (zipEntry != null) { 91 | if (zipEntry.isDirectory) { 92 | gradleProfilerDir.resolve(zipEntry.name.dropLeadingDir).also { it.mkdirs() } 93 | } else { 94 | gradleProfilerDir.resolve(zipEntry.name.dropLeadingDir).outputStream().buffered().use { 95 | zip.copyTo(it) 96 | } 97 | } 98 | downloadedLength += zipEntry.compressedSize 99 | logger.debug("..{}%", downloadedLength * 100 / contentLength) 100 | zip.closeEntry() 101 | zipEntry = zip.nextEntry 102 | } 103 | } 104 | logger.debug("\n") 105 | } 106 | gradleProfilerBin.get().asFile.setExecutable(true) 107 | logger.info("Finished downloading gradle-profiler") 108 | } 109 | } -------------------------------------------------------------------------------- /buildSrc/src/main/kotlin/blogpost/PerformanceTask.kt: -------------------------------------------------------------------------------- 1 | package org.jetbrains.kotlin.k2.blogpost 2 | 3 | import org.gradle.api.DefaultTask 4 | import java.io.File 5 | import org.gradle.api.Project 6 | import org.gradle.api.file.DirectoryProperty 7 | import org.gradle.api.file.ProjectLayout 8 | import org.gradle.api.file.RegularFile 9 | import org.gradle.api.file.RegularFileProperty 10 | import org.gradle.api.model.ObjectFactory 11 | import org.gradle.api.provider.ListProperty 12 | import org.gradle.api.provider.Property 13 | import org.gradle.api.provider.Provider 14 | import org.gradle.api.provider.ProviderFactory 15 | import org.gradle.api.tasks.* 16 | import org.gradle.kotlin.dsl.listProperty 17 | import org.gradle.kotlin.dsl.property 18 | import org.gradle.kotlin.dsl.register 19 | import javax.inject.Inject 20 | import kotlin.concurrent.thread 21 | 22 | abstract class PerformanceTask @Inject constructor( 23 | providerFactory: ProviderFactory, 24 | objectFactory: ObjectFactory, 25 | private val projectLayout: ProjectLayout, 26 | ) : DefaultTask() { 27 | 28 | @get:Input 29 | abstract val kotlinVersion: Property 30 | 31 | private val gradleTask = providerFactory.gradleProperty("scenario.task").orElse("assemble") 32 | 33 | @Nested 34 | val scenarios: ListProperty = objectFactory.listProperty().value( 35 | gradleTask.zip(kotlinVersion) { gradleTask, kotlinVersion -> 36 | val runUsing = project.providers.gradleProperty("scenario.run.using").orNull ?: "cli" 37 | listOf( 38 | Scenario( 39 | "clean_build", 40 | cleanTasks = listOf("clean"), 41 | tasks = listOf(gradleTask), 42 | projectDir = project.projectDir, 43 | kotlinVersion = kotlinVersion, 44 | runUsing = runUsing, 45 | ), 46 | Scenario( 47 | "incremental_non_abi_build", 48 | tasks = listOf(gradleTask), 49 | projectDir = project.projectDir, 50 | nonAbiChanges = project.providers.gradleProperty("scenario.non.abi.changes").orNull?.split(",") 51 | ?: throw IllegalArgumentException( 52 | createErrorMessageForMissedProperty( 53 | propertyName = "scenario.non.abi.changes", 54 | scenarioName = "incremental_non_abi_build" 55 | ) 56 | ), 57 | kotlinVersion = kotlinVersion, 58 | runUsing = runUsing, 59 | ), 60 | Scenario( 61 | "incremental_abi_build", 62 | tasks = listOf(gradleTask), 63 | projectDir = project.projectDir, 64 | abiChanges = project.providers.gradleProperty("scenario.abi.changes").orNull 65 | ?: throw IllegalArgumentException( 66 | createErrorMessageForMissedProperty( 67 | propertyName = "scenario.abi.changes", 68 | scenarioName = "incremental_abi_build" 69 | ) 70 | ), 71 | kotlinVersion = kotlinVersion, 72 | runUsing = runUsing, 73 | ), 74 | ) 75 | } 76 | ) 77 | 78 | 79 | @get:Nested 80 | val testProject: Provider = objectFactory.property().value( 81 | providerFactory.provider { TestProject.createTestProject(project, logger) } 82 | ) 83 | 84 | @get:Optional 85 | @get:InputFile 86 | abstract val scenarioFile: RegularFileProperty 87 | 88 | @get:InputDirectory 89 | abstract val gradleProfilerDir: DirectoryProperty 90 | 91 | private val gradleProfilerBin: Provider = gradleProfilerDir.map { 92 | it.dir("bin") 93 | .run { 94 | if (System.getProperty("os.name").contains("windows", ignoreCase = true)) { 95 | file("gradle-profiler.bat") 96 | } else { 97 | file("gradle-profiler") 98 | } 99 | } 100 | } 101 | 102 | private val generatedScenarioFile: Provider = projectLayout 103 | .buildDirectory 104 | .dir("generated-scenarios/$name") 105 | .zip(kotlinVersion) { dir, kotlinVersion -> 106 | dir.file("benchmark_${kotlinVersion.replace(".", "_")}.scenarios") 107 | } 108 | 109 | 110 | @TaskAction 111 | fun run() { 112 | val scenario = createScenariosIfNeed() 113 | testProject.get().checkoutProjectFromGit(logger) 114 | runBenchmark(scenario) 115 | } 116 | 117 | private fun createScenariosIfNeed(): File { 118 | return scenarioFile.orNull?.asFile ?: generatedScenarioFile.get().asFile.also { file -> 119 | file.parentFile.mkdirs() 120 | file.createNewFile() 121 | file.outputStream().bufferedWriter().use { writer -> 122 | scenarios.get().forEach { 123 | it.printScenario(writer) 124 | } 125 | writer.flush() 126 | } 127 | } 128 | } 129 | 130 | companion object { 131 | @JvmStatic 132 | fun registerPerformanceTask( 133 | project: Project, 134 | benchmarkName: String, 135 | testedKotlinVersion: String, 136 | configure: PerformanceTask.() -> Unit = {} 137 | ): TaskProvider { 138 | return project.tasks.register(benchmarkName) { 139 | description = "Run $benchmarkName with Kotlin version $testedKotlinVersion" 140 | group = "benchmarks" 141 | kotlinVersion.value(testedKotlinVersion).finalizeValue() 142 | configure(this) 143 | } 144 | } 145 | 146 | private fun createErrorMessageForMissedProperty(propertyName: String, scenarioName: String) = 147 | "Unable to create \'$scenarioName\' scenario for performance test.\n" + 148 | "Please provide path to a project file with \'$propertyName\' property." 149 | } 150 | 151 | private fun runBenchmark(scenarioFile: File) { 152 | logger.info("Staring benchmark") 153 | val workDirectory = projectLayout.buildDirectory.dir("benchmarkRuns").get().asFile.also { 154 | it.mkdirs() 155 | } 156 | //files in output directory will be overriden by gradle profile 157 | val gradleProfilerOutputDir = 158 | "${testProject.get().projectDir.name}-${kotlinVersion.get()}-profile-out".replace(".", "-") 159 | 160 | val profilerProcessBuilder = ProcessBuilder() 161 | .directory(workDirectory) 162 | .command( 163 | gradleProfilerBin.get().asFile.absolutePath, 164 | "--benchmark", 165 | "--project-dir", 166 | testProject.get().projectDir.absolutePath, 167 | "--output-dir", 168 | gradleProfilerOutputDir, 169 | "--scenario-file", 170 | scenarioFile.absolutePath, 171 | ) 172 | .also { 173 | // Required, so 'gradle-profiler' will use toolchain JDK instead of current user one 174 | it.environment()["JAVA_HOME"] = System.getProperty("java.home") 175 | it.environment()["kotlin_version"] = kotlinVersion.get() 176 | } 177 | 178 | runBenchmarksForKotlinVersion(profilerProcessBuilder, workDirectory.resolve(gradleProfilerOutputDir)) 179 | } 180 | 181 | private fun runBenchmarksForKotlinVersion(profilerProcessBuilder: ProcessBuilder, gradleProfilerOutputDir: File) { 182 | val logFile = gradleProfilerOutputDir.resolve("profile.log") 183 | logger.lifecycle("Benchmarks log will be written at file://${logFile.absolutePath}") 184 | 185 | // Gradle benchmark tool prints both to file and to stdout. 186 | // Discarding (or redirecting to any file) standart output also fixes "hanging" on Windows hosts 187 | profilerProcessBuilder.redirectOutput(ProcessBuilder.Redirect.DISCARD) 188 | 189 | val profilerProcess = profilerProcessBuilder.start() 190 | // Stop profiler on script stop 191 | Runtime.getRuntime().addShutdownHook(Thread { 192 | profilerProcess.destroy() 193 | }) 194 | 195 | // stderr reader 196 | val stdErrPrinter = thread { 197 | val reader = profilerProcess.errorStream.bufferedReader() 198 | reader.lines().forEach { logger.error(it) } 199 | } 200 | 201 | profilerProcess.waitFor() 202 | stdErrPrinter.join() // wait for stderr printer to finish printing errors 203 | 204 | if (profilerProcess.exitValue() != 0) { 205 | throw IllegalStateException("Benchmarks finished with non-zero exit code: ${profilerProcess.exitValue()}") 206 | } else { 207 | logger.lifecycle("Benchmarks finished successfully.") 208 | } 209 | } 210 | } -------------------------------------------------------------------------------- /buildSrc/src/main/kotlin/blogpost/Scenario.kt: -------------------------------------------------------------------------------- 1 | package org.jetbrains.kotlin.k2.blogpost 2 | 3 | import org.gradle.api.tasks.Input 4 | import org.gradle.api.tasks.InputDirectory 5 | import org.gradle.api.tasks.Nested 6 | import org.gradle.api.tasks.Optional 7 | import java.io.BufferedWriter 8 | import java.io.File 9 | 10 | data class Scenario( 11 | @get:Input 12 | val name: String, 13 | @get:Input 14 | val kotlinVersion: String, 15 | @get:Input 16 | @get:Optional 17 | val cleanTasks: List? = null, 18 | @get:Input 19 | val tasks: List = listOf("assemble"), 20 | @get:InputDirectory 21 | val projectDir: File, 22 | @get:Input 23 | val warmUpRounds: Int = 10, 24 | @get:Input 25 | val executionRounds: Int = 10, 26 | @get:Input 27 | val additionGradleArguments: List = emptyList(), 28 | @get:Input 29 | val runUsing: String, 30 | @get:Input 31 | @get:Optional 32 | val abiChanges: String? = null, 33 | @get:Nested 34 | @get:Optional 35 | val gitCheckout: GitCheckout? = null, 36 | @get:Input 37 | @get:Optional 38 | val nonAbiChanges: List? = null, 39 | ) { 40 | fun printScenario(writer: BufferedWriter) { 41 | writer.append("$name {\n") 42 | cleanTasks?.let { tasks -> 43 | writer.append(tasks.joinToString( 44 | separator = ", ", 45 | prefix = "cleanup-tasks = [", 46 | postfix = "]\n", 47 | transform = { "\"$it\"" })) 48 | } 49 | writer.append( 50 | tasks.joinToString( 51 | separator = ", ", 52 | prefix = "tasks = [", 53 | postfix = "]\n", 54 | transform = { "\"$it\"" }) 55 | ) 56 | writer.append("warm-ups = $warmUpRounds\n") 57 | writer.append("iterations = $executionRounds\n") 58 | writer.append("clear-build-cache-before = SCENARIO\n") 59 | writer.append("clear-transform-cache-before = SCENARIO\n") 60 | abiChanges?.let { 61 | writer.append("apply-abi-change-to = \"$it\"\n") 62 | } 63 | nonAbiChanges?.let { changes -> 64 | writer.append(changes.joinToString( 65 | separator = ", ", 66 | prefix = "apply-non-abi-change-to = [", 67 | postfix = "]\n", 68 | transform = { "\"$it\"" })) 69 | } 70 | gitCheckout?.let { (cleanup, build) -> 71 | writer.append("git-checkout = {\ncleanup = \"$cleanup\"\nbuild = \"$build\"\n}\n") 72 | } 73 | 74 | writer.append("run-using = $runUsing\n") 75 | 76 | writer.append("gradle-args = [\"--no-build-cache\", \"-Pkotlin.build.report.output=JSON,FILE\", \"-Pkotlin.daemon.jvmargs=-Xmx=4G\", " + 77 | "\"-Pkotlin.build.report.json.directory=${escapeSymbolsForWindows(projectDir.resolve("reports/$kotlinVersion/$name").path)}\", " + 78 | "\"-Pkotlin.build.report.file.output_dir=${escapeSymbolsForWindows(projectDir.resolve("reports/$kotlinVersion/$name").path)}\", " + 79 | "\"-Pkotlin_version=$kotlinVersion\"") 80 | writer.append(additionGradleArguments.joinToString(separator = ", ", prefix = ", ", transform = { "\"$it\"" })) 81 | writer.append("]\n") 82 | 83 | writer.append("}\n") 84 | } 85 | 86 | private fun escapeSymbolsForWindows(path: String) = path.replace("\\", "\\\\") 87 | } 88 | 89 | data class GitCheckout( 90 | @get:Input 91 | val cleanup: String, 92 | @get:Input 93 | val build: String, 94 | ) 95 | -------------------------------------------------------------------------------- /buildSrc/src/main/kotlin/blogpost/TestProject.kt: -------------------------------------------------------------------------------- 1 | package org.jetbrains.kotlin.k2.blogpost 2 | 3 | import java.io.File 4 | import org.eclipse.jgit.api.Git 5 | import org.eclipse.jgit.api.ResetCommand 6 | import org.eclipse.jgit.lib.TextProgressMonitor 7 | import org.gradle.api.Project 8 | import org.gradle.api.logging.Logger 9 | import org.gradle.api.tasks.Input 10 | import org.gradle.api.tasks.Internal 11 | import org.gradle.api.tasks.Optional 12 | 13 | class TestProject( 14 | @get:Internal 15 | val projectDir: File, 16 | 17 | @get:Optional 18 | @get:Input 19 | val projectGitUrl: String? = null, 20 | 21 | @get:Optional 22 | @get:Input 23 | val gitCommitSha: String? = null, 24 | ) { 25 | 26 | @get:Input 27 | val projectDirAsString = projectDir.absolutePath 28 | 29 | fun checkoutProjectFromGit(logger: Logger) { 30 | if (projectGitUrl == null) return 31 | 32 | require(gitCommitSha != null) { "Git commit SHA is required" } 33 | 34 | logger.info("Git project will be checked out into \'${projectDir.path}\'") 35 | val git = if (projectDir.exists()) { 36 | logger.info("Repository is available, resetting it state") 37 | Git.open(projectDir).also { 38 | it.reset() 39 | .setMode(ResetCommand.ResetType.HARD) 40 | .setProgressMonitor(gitOperationsPrinter) 41 | .call() 42 | } 43 | } else { 44 | logger.info("Running git checkout for $projectGitUrl") 45 | projectDir.mkdirs() 46 | Git.cloneRepository() 47 | .setDirectory(projectDir) 48 | .setCloneSubmodules(true) 49 | .setProgressMonitor(gitOperationsPrinter) 50 | .setURI(projectGitUrl) 51 | .call() 52 | } 53 | 54 | git.checkout() 55 | .setName(gitCommitSha) 56 | .setProgressMonitor(gitOperationsPrinter) 57 | .call() 58 | } 59 | 60 | companion object { 61 | private val gitOperationsPrinter = TextProgressMonitor() 62 | private const val PROJECT_PATH_PROPERTY = "project.path" 63 | private const val PROJECT_GIT_URL_PROPERTY = "project.git.url" 64 | private const val PROJECT_GIT_COMMIT_SHA_PROPERTY = "project.git.commit.sha" 65 | 66 | private const val errorMessage = "Unable to initialize project for performance test.\n" + 67 | "Please either provide path to existed project with \'$PROJECT_PATH_PROPERTY\' property \n" + 68 | "or git information with \'$PROJECT_GIT_URL_PROPERTY\' and \'$PROJECT_GIT_COMMIT_SHA_PROPERTY\' properties \n" + 69 | "to check out it." 70 | 71 | internal fun createTestProject(project: Project, logger: Logger): TestProject { 72 | val providers = project.providers 73 | val projectPath = providers.gradleProperty(PROJECT_PATH_PROPERTY).orNull 74 | if (projectPath != null) { 75 | logger.info("Use existing project located at \'$projectPath\' for performance test") 76 | return TestProject(File(projectPath)) 77 | } 78 | 79 | val projectGitUrl = providers.gradleProperty(PROJECT_GIT_URL_PROPERTY).orNull 80 | ?: throw IllegalArgumentException(errorMessage) 81 | 82 | val gitCommitSha = providers.gradleProperty(PROJECT_GIT_COMMIT_SHA_PROPERTY).orNull ?: throw IllegalArgumentException(errorMessage) 83 | val projectDir = providers.gradleProperty("project.git.directory").orNull?.let { File(it) } 84 | ?: project.layout.buildDirectory 85 | .dir(projectGitUrl.gitName()) 86 | .get().asFile 87 | return TestProject(projectDir, projectGitUrl, gitCommitSha) 88 | } 89 | 90 | fun String.gitName(): String { 91 | return substringBefore(".git").substringAfterLast("/") 92 | } 93 | } 94 | } -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | # Example how to define the project path for benchmark 2 | #project.path=/path/to/your/project/for/unix 3 | #project.path=D:\\path\\to\\your\\project\\for\\windows 4 | 5 | # Example how to define the project using git repository 6 | #project.git.url=https://github.com/JetBrains/kotlin.git 7 | #project.git.commit.sha=ffef1630dde48bac130f9fb693c36e4bffd3b78a 8 | 9 | # Example how to re-define some scenario values (for example for https://github.com/JetBrains/Exposed) 10 | #scenario.non.abi.changes=exposed-core/src/main/kotlin/org/jetbrains/exposed/dao/id/EntityID.kt 11 | #scenario.abi.changes=exposed-core/src/main/kotlin/org/jetbrains/exposed/dao/id/EntityID.kt 12 | #scenario.task=assemble 13 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kotlin/k2-performance-metrics/5b9b4fb0f73c2ac2426f6a34a0c25898713491d0/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-8.8-bin.zip 4 | networkTimeout=10000 5 | validateDistributionUrl=true 6 | zipStoreBase=GRADLE_USER_HOME 7 | zipStorePath=wrapper/dists 8 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # 4 | # Copyright © 2015-2021 the original authors. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | ############################################################################## 20 | # 21 | # Gradle start up script for POSIX generated by Gradle. 22 | # 23 | # Important for running: 24 | # 25 | # (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is 26 | # noncompliant, but you have some other compliant shell such as ksh or 27 | # bash, then to run this script, type that shell name before the whole 28 | # command line, like: 29 | # 30 | # ksh Gradle 31 | # 32 | # Busybox and similar reduced shells will NOT work, because this script 33 | # requires all of these POSIX shell features: 34 | # * functions; 35 | # * expansions «$var», «${var}», «${var:-default}», «${var+SET}», 36 | # «${var#prefix}», «${var%suffix}», and «$( cmd )»; 37 | # * compound commands having a testable exit status, especially «case»; 38 | # * various built-in commands including «command», «set», and «ulimit». 39 | # 40 | # Important for patching: 41 | # 42 | # (2) This script targets any POSIX shell, so it avoids extensions provided 43 | # by Bash, Ksh, etc; in particular arrays are avoided. 44 | # 45 | # The "traditional" practice of packing multiple parameters into a 46 | # space-separated string is a well documented source of bugs and security 47 | # problems, so this is (mostly) avoided, by progressively accumulating 48 | # options in "$@", and eventually passing that to Java. 49 | # 50 | # Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, 51 | # and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; 52 | # see the in-line comments for details. 53 | # 54 | # There are tweaks for specific operating systems such as AIX, CygWin, 55 | # Darwin, MinGW, and NonStop. 56 | # 57 | # (3) This script is generated from the Groovy template 58 | # https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt 59 | # within the Gradle project. 60 | # 61 | # You can find Gradle at https://github.com/gradle/gradle/. 62 | # 63 | ############################################################################## 64 | 65 | # Attempt to set APP_HOME 66 | 67 | # Resolve links: $0 may be a link 68 | app_path=$0 69 | 70 | # Need this for daisy-chained symlinks. 71 | while 72 | APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path 73 | [ -h "$app_path" ] 74 | do 75 | ls=$( ls -ld "$app_path" ) 76 | link=${ls#*' -> '} 77 | case $link in #( 78 | /*) app_path=$link ;; #( 79 | *) app_path=$APP_HOME$link ;; 80 | esac 81 | done 82 | 83 | # This is normally unused 84 | # shellcheck disable=SC2034 85 | APP_BASE_NAME=${0##*/} 86 | # Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) 87 | APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit 88 | 89 | # Use the maximum available, or set MAX_FD != -1 to use that value. 90 | MAX_FD=maximum 91 | 92 | warn () { 93 | echo "$*" 94 | } >&2 95 | 96 | die () { 97 | echo 98 | echo "$*" 99 | echo 100 | exit 1 101 | } >&2 102 | 103 | # OS specific support (must be 'true' or 'false'). 104 | cygwin=false 105 | msys=false 106 | darwin=false 107 | nonstop=false 108 | case "$( uname )" in #( 109 | CYGWIN* ) cygwin=true ;; #( 110 | Darwin* ) darwin=true ;; #( 111 | MSYS* | MINGW* ) msys=true ;; #( 112 | NONSTOP* ) nonstop=true ;; 113 | esac 114 | 115 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 116 | 117 | 118 | # Determine the Java command to use to start the JVM. 119 | if [ -n "$JAVA_HOME" ] ; then 120 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 121 | # IBM's JDK on AIX uses strange locations for the executables 122 | JAVACMD=$JAVA_HOME/jre/sh/java 123 | else 124 | JAVACMD=$JAVA_HOME/bin/java 125 | fi 126 | if [ ! -x "$JAVACMD" ] ; then 127 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 128 | 129 | Please set the JAVA_HOME variable in your environment to match the 130 | location of your Java installation." 131 | fi 132 | else 133 | JAVACMD=java 134 | if ! command -v java >/dev/null 2>&1 135 | then 136 | die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 137 | 138 | Please set the JAVA_HOME variable in your environment to match the 139 | location of your Java installation." 140 | fi 141 | fi 142 | 143 | # Increase the maximum file descriptors if we can. 144 | if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then 145 | case $MAX_FD in #( 146 | max*) 147 | # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. 148 | # shellcheck disable=SC2039,SC3045 149 | MAX_FD=$( ulimit -H -n ) || 150 | warn "Could not query maximum file descriptor limit" 151 | esac 152 | case $MAX_FD in #( 153 | '' | soft) :;; #( 154 | *) 155 | # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. 156 | # shellcheck disable=SC2039,SC3045 157 | ulimit -n "$MAX_FD" || 158 | warn "Could not set maximum file descriptor limit to $MAX_FD" 159 | esac 160 | fi 161 | 162 | # Collect all arguments for the java command, stacking in reverse order: 163 | # * args from the command line 164 | # * the main class name 165 | # * -classpath 166 | # * -D...appname settings 167 | # * --module-path (only if needed) 168 | # * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. 169 | 170 | # For Cygwin or MSYS, switch paths to Windows format before running java 171 | if "$cygwin" || "$msys" ; then 172 | APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) 173 | CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) 174 | 175 | JAVACMD=$( cygpath --unix "$JAVACMD" ) 176 | 177 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 178 | for arg do 179 | if 180 | case $arg in #( 181 | -*) false ;; # don't mess with options #( 182 | /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath 183 | [ -e "$t" ] ;; #( 184 | *) false ;; 185 | esac 186 | then 187 | arg=$( cygpath --path --ignore --mixed "$arg" ) 188 | fi 189 | # Roll the args list around exactly as many times as the number of 190 | # args, so each arg winds up back in the position where it started, but 191 | # possibly modified. 192 | # 193 | # NB: a `for` loop captures its iteration list before it begins, so 194 | # changing the positional parameters here affects neither the number of 195 | # iterations, nor the values presented in `arg`. 196 | shift # remove old arg 197 | set -- "$@" "$arg" # push replacement arg 198 | done 199 | fi 200 | 201 | 202 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 203 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' 204 | 205 | # Collect all arguments for the java command: 206 | # * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, 207 | # and any embedded shellness will be escaped. 208 | # * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be 209 | # treated as '${Hostname}' itself on the command line. 210 | 211 | set -- \ 212 | "-Dorg.gradle.appname=$APP_BASE_NAME" \ 213 | -classpath "$CLASSPATH" \ 214 | org.gradle.wrapper.GradleWrapperMain \ 215 | "$@" 216 | 217 | # Stop when "xargs" is not available. 218 | if ! command -v xargs >/dev/null 2>&1 219 | then 220 | die "xargs is not available" 221 | fi 222 | 223 | # Use "xargs" to parse quoted args. 224 | # 225 | # With -n1 it outputs one arg per line, with the quotes and backslashes removed. 226 | # 227 | # In Bash we could simply go: 228 | # 229 | # readarray ARGS < <( xargs -n1 <<<"$var" ) && 230 | # set -- "${ARGS[@]}" "$@" 231 | # 232 | # but POSIX shell has neither arrays nor command substitution, so instead we 233 | # post-process each arg (as a line of input to sed) to backslash-escape any 234 | # character that might be a shell metacharacter, then use eval to reverse 235 | # that process (while maintaining the separation between arguments), and wrap 236 | # the whole thing up as a single "set" statement. 237 | # 238 | # This will of course break if any of these variables contains a newline or 239 | # an unmatched quote. 240 | # 241 | 242 | eval "set -- $( 243 | printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | 244 | xargs -n1 | 245 | sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | 246 | tr '\n' ' ' 247 | )" '"$@"' 248 | 249 | exec "$JAVACMD" "$@" 250 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%"=="" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%"=="" set DIRNAME=. 29 | @rem This is normally unused 30 | set APP_BASE_NAME=%~n0 31 | set APP_HOME=%DIRNAME% 32 | 33 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 34 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 35 | 36 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 37 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 38 | 39 | @rem Find java.exe 40 | if defined JAVA_HOME goto findJavaFromJavaHome 41 | 42 | set JAVA_EXE=java.exe 43 | %JAVA_EXE% -version >NUL 2>&1 44 | if %ERRORLEVEL% equ 0 goto execute 45 | 46 | echo. 47 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 48 | echo. 49 | echo Please set the JAVA_HOME variable in your environment to match the 50 | echo location of your Java installation. 51 | 52 | goto fail 53 | 54 | :findJavaFromJavaHome 55 | set JAVA_HOME=%JAVA_HOME:"=% 56 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 57 | 58 | if exist "%JAVA_EXE%" goto execute 59 | 60 | echo. 61 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 62 | echo. 63 | echo Please set the JAVA_HOME variable in your environment to match the 64 | echo location of your Java installation. 65 | 66 | goto fail 67 | 68 | :execute 69 | @rem Setup the command line 70 | 71 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 72 | 73 | 74 | @rem Execute Gradle 75 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 76 | 77 | :end 78 | @rem End local scope for the variables with windows NT shell 79 | if %ERRORLEVEL% equ 0 goto mainEnd 80 | 81 | :fail 82 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 83 | rem the _cmd.exe /c_ return code! 84 | set EXIT_CODE=%ERRORLEVEL% 85 | if %EXIT_CODE% equ 0 set EXIT_CODE=1 86 | if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% 87 | exit /b %EXIT_CODE% 88 | 89 | :mainEnd 90 | if "%OS%"=="Windows_NT" endlocal 91 | 92 | :omega 93 | -------------------------------------------------------------------------------- /settings.gradle.kts: -------------------------------------------------------------------------------- 1 | pluginManagement { 2 | repositories { 3 | google() 4 | mavenCentral() 5 | gradlePluginPortal() 6 | } 7 | } 8 | plugins { 9 | id("org.gradle.toolchains.foojay-resolver-convention") version "0.5.0" 10 | } 11 | 12 | --------------------------------------------------------------------------------