├── bazel_support ├── BUILD ├── constants.bzl └── repositories.bzl ├── samples ├── .gitignore ├── protobuf-sync-code-gen │ ├── settings.gradle.kts │ ├── gitquery.yml │ ├── README.md │ ├── src │ │ └── main │ │ │ └── proto │ │ │ └── examples │ │ │ └── addressbook.proto │ └── build.gradle.kts ├── protobuf-sync-groovy-dsl │ ├── settings.gradle │ ├── README.md │ ├── build.gradle │ ├── gitquery.yml │ └── src │ │ └── main │ │ └── proto │ │ └── examples │ │ └── addressbook.proto ├── sample1.yml ├── sample2-generate-nested.yml └── sample2-generate-flat.yml ├── cli ├── gradle.properties ├── build.gradle.kts └── src │ └── main │ └── kotlin │ └── com │ └── tinder │ └── gitquery │ └── cli │ └── GitQueryCli.kt ├── core ├── gradle.properties ├── src │ ├── main │ │ └── kotlin │ │ │ └── com │ │ │ └── tinder │ │ │ └── gitquery │ │ │ └── core │ │ │ ├── GitQueryVersion.kt │ │ │ ├── config │ │ │ ├── GitQueryConfigSchema.kt │ │ │ ├── GitQueryInitConfig.kt │ │ │ └── GitQueryConfig.kt │ │ │ ├── utils │ │ │ ├── HashMapExtensions.kt │ │ │ ├── PathUtils.kt │ │ │ ├── ShellUtils.kt │ │ │ └── RepoUtils.kt │ │ │ ├── GitQuerySync.kt │ │ │ └── GitQueryInit.kt │ └── test │ │ └── kotlin │ │ └── com │ │ └── tinder │ │ └── gitquery │ │ └── core │ │ ├── GitQueryConfigTest.kt │ │ ├── GitQueryInitTest.kt │ │ └── GitQuerySyncTest.kt ├── BUILD.bazel └── build.gradle.kts ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradle-plugin ├── gradle.properties ├── src │ ├── main │ │ └── kotlin │ │ │ └── com │ │ │ └── tinder │ │ │ └── gitquery │ │ │ ├── GitQueryInitExtension.kt │ │ │ ├── GitQueryPlugin.kt │ │ │ ├── GitQuerySyncExtension.kt │ │ │ ├── GitQueryInitTask.kt │ │ │ └── GitQuerySyncTask.kt │ └── test │ │ └── kotlin │ │ └── com │ │ └── tinder │ │ └── gitquery │ │ ├── GitQueryInitTaskTest.kt │ │ └── GitQuerySyncTaskTest.kt └── build.gradle.kts ├── settings.gradle.kts ├── gitquery ├── install ├── .github └── workflows │ ├── bazel.yml │ ├── gradle.yml │ └── samples_gradle.yml ├── WORKSPACE ├── gradle.properties ├── publish ├── BUILD.bazel ├── LICENSE.txt ├── .gitignore ├── gradlew.bat ├── gradlew └── README.md /bazel_support/BUILD: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /samples/.gitignore: -------------------------------------------------------------------------------- 1 | */gradle/* 2 | */gradlew 3 | */gradlew.bat -------------------------------------------------------------------------------- /cli/gradle.properties: -------------------------------------------------------------------------------- 1 | POM_ARTIFACT_ID=cli 2 | POM_NAME=GitQuery CLI 3 | POM_PACKAGING=jar 4 | -------------------------------------------------------------------------------- /core/gradle.properties: -------------------------------------------------------------------------------- 1 | POM_ARTIFACT_ID=core 2 | POM_NAME=GitQuery Core 3 | POM_PACKAGING=jar 4 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tinder/GitQuery/HEAD/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle-plugin/gradle.properties: -------------------------------------------------------------------------------- 1 | POM_ARTIFACT_ID=gradle-plugin 2 | POM_NAME=GitQuery Gradle Plugin 3 | POM_PACKAGING=jar 4 | -------------------------------------------------------------------------------- /settings.gradle.kts: -------------------------------------------------------------------------------- 1 | include( 2 | "cli", 3 | "core", 4 | "gradle-plugin" 5 | ) 6 | 7 | pluginManagement { 8 | repositories { 9 | gradlePluginPortal() 10 | jcenter() 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /core/src/main/kotlin/com/tinder/gitquery/core/GitQueryVersion.kt: -------------------------------------------------------------------------------- 1 | package com.tinder.gitquery.core 2 | 3 | /** 4 | * This file was generated by ./build-support/bin/write-version-class 5 | */ 6 | 7 | const val GIT_QUERY_VERSION = "3.1.0" 8 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /samples/protobuf-sync-code-gen/settings.gradle.kts: -------------------------------------------------------------------------------- 1 | pluginManagement { 2 | repositories { 3 | if (System.getenv("CI") == "true") { 4 | mavenLocal() 5 | } 6 | // for GA versions 7 | gradlePluginPortal() 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /samples/protobuf-sync-groovy-dsl/settings.gradle: -------------------------------------------------------------------------------- 1 | pluginManagement { 2 | repositories { 3 | if (System.getenv("CI") == "true") { 4 | mavenLocal() 5 | } 6 | // for GA versions 7 | gradlePluginPortal() 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /samples/protobuf-sync-groovy-dsl/README.md: -------------------------------------------------------------------------------- 1 | ## Gradle Proto to Kotlin Code Gen 2 | 3 | This examples shows how to generate a gitquery config, and use it to sync the files. 4 | 5 | ### Building 6 | 7 | To build the code simply run: 8 | 9 | path/to/gradle gitQueryInit build 10 | 11 | 12 | -------------------------------------------------------------------------------- /core/src/main/kotlin/com/tinder/gitquery/core/config/GitQueryConfigSchema.kt: -------------------------------------------------------------------------------- 1 | /* 2 | * © 2019 Match Group, LLC. 3 | */ 4 | 5 | package com.tinder.gitquery.core.config 6 | 7 | /** 8 | * A model fof the schema attribute of the config file. 9 | */ 10 | data class GitQueryConfigSchema( 11 | var version: String = "1" 12 | ) 13 | -------------------------------------------------------------------------------- /gitquery: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # © 2019 Match Group, LLC. 4 | # 5 | # GitQuery - Command Line Interface 6 | 7 | scriptDir=$( cd "$(dirname "${BASH_SOURCE[0]}")" || exit 1 ; pwd -P ) 8 | 9 | cd "$scriptDir" || exit 1 10 | 11 | ([ -f ./cli/build/install/cli/bin/cli ] || ./install) && ./cli/build/install/cli/bin/cli "$@" 12 | -------------------------------------------------------------------------------- /core/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_kotlin//kotlin:kotlin.bzl", "kt_jvm_library") 2 | 3 | kt_jvm_library( 4 | name = "core", 5 | srcs = glob(["src/main/**/*.kt"], allow_empty = False), 6 | deps = [ 7 | "@maven//:org_yaml_snakeyaml" 8 | ], 9 | module_name = "core", 10 | visibility = ["//visibility:public"] 11 | ) 12 | -------------------------------------------------------------------------------- /install: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # © 2019 Match Group, LLC. 4 | # 5 | # GitQuery - Install Command Line Interface 6 | 7 | scriptDir=$( cd "$(dirname "${BASH_SOURCE[0]}")" || exit 1 ; pwd -P ) 8 | 9 | cd "$scriptDir" || exit 1 10 | 11 | ./gradlew --quiet clean installDist 12 | 13 | ln -sF $(pwd)/cli/build/install/cli/bin/cli /usr/local/bin/gitquery 14 | -------------------------------------------------------------------------------- /cli/build.gradle.kts: -------------------------------------------------------------------------------- 1 | /* 2 | * © 2019 Match Group, LLC. 3 | */ 4 | 5 | plugins { 6 | application 7 | kotlin("jvm") 8 | id("com.vanniktech.maven.publish") 9 | } 10 | 11 | application { 12 | mainClass.set("com.tinder.gitquery.cli.GitQueryCliKt") 13 | } 14 | 15 | dependencies { 16 | kotlinLibrary() 17 | implementation(project(":core")) 18 | implementation(Libs.clikt) 19 | } 20 | -------------------------------------------------------------------------------- /samples/protobuf-sync-groovy-dsl/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id("java") 3 | id("com.tinder.gitquery") version "3.0.12" 4 | } 5 | 6 | def protoDir = "src/main/proto" 7 | 8 | gitQuery { 9 | autoSync = true 10 | configFile = "gitquery.yml" 11 | outputDir = protoDir 12 | repoDir = "tmp/.gitquery" 13 | } 14 | 15 | repositories { 16 | jcenter() 17 | if (System.getenv("CI") == "true") { 18 | mavenLocal() 19 | } 20 | mavenCentral() 21 | } 22 | -------------------------------------------------------------------------------- /.github/workflows/bazel.yml: -------------------------------------------------------------------------------- 1 | name: Bazel 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | build: 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - name: Setup Java JDK 15 | uses: actions/setup-java@v1.4.3 16 | with: 17 | java-version: 11 18 | id: java 19 | - uses: actions/checkout@v2 20 | - name: Build Deploy Jar 21 | run: bazelisk build //:GitQuery 22 | -------------------------------------------------------------------------------- /bazel_support/constants.bzl: -------------------------------------------------------------------------------- 1 | """ 2 | Constants used in GitQuery configs 3 | """ 4 | 5 | RULES_JVM_EXTERNAL_TAG = "6.1" 6 | RULES_JVM_EXTERNAL_SHA = "42a6d48eb2c08089961c715a813304f30dc434df48e371ebdd868fc3636f0e82" 7 | 8 | RULES_KOTLIN_VERSION = "v1.9.5" 9 | RULES_KOTLIN_SHA = "34e8c0351764b71d78f76c8746e98063979ce08dcf1a91666f3f3bc2949a533d" 10 | 11 | MAVEN_ARTIFACTS = [ 12 | "org.yaml:snakeyaml:1.27", 13 | "com.github.ajalt.clikt:clikt-jvm:3.1.0", 14 | "org.seleniumhq.selenium:selenium-java:4.16.1", 15 | ] 16 | -------------------------------------------------------------------------------- /samples/protobuf-sync-code-gen/gitquery.yml: -------------------------------------------------------------------------------- 1 | !!com.tinder.gitquery.core.config.GitQueryConfig 2 | branch: master 3 | cleanOutput: true 4 | commits: &id001 {} 5 | extra: *id001 6 | files: 7 | examples: 8 | addressbook.proto: v3.14.0 9 | initConfig: 10 | excludeGlobs: 11 | - '**/generated/**/*.proto' 12 | flatFiles: false 13 | includeGlobs: 14 | - '**/examples/addressbook.proto' 15 | revision: v3.14.0 16 | outputDir: src/main/proto 17 | remote: git@github.com:protocolbuffers/protobuf.git 18 | repoDir: tmp/.gitquery 19 | schema: 20 | version: '1' 21 | -------------------------------------------------------------------------------- /samples/protobuf-sync-groovy-dsl/gitquery.yml: -------------------------------------------------------------------------------- 1 | !!com.tinder.gitquery.core.config.GitQueryConfig 2 | branch: master 3 | cleanOutput: true 4 | commits: &id001 {} 5 | extra: *id001 6 | files: 7 | examples: 8 | addressbook.proto: v3.14.0 9 | initConfig: 10 | excludeGlobs: 11 | - '**/generated/**/*.proto' 12 | flatFiles: false 13 | includeGlobs: 14 | - '**/examples/addressbook.proto' 15 | revision: v3.14.0 16 | outputDir: src/main/proto 17 | remote: git@github.com:protocolbuffers/protobuf.git 18 | repoDir: tmp/.gitquery 19 | schema: 20 | version: '1' 21 | -------------------------------------------------------------------------------- /core/build.gradle.kts: -------------------------------------------------------------------------------- 1 | /* 2 | * © 2019 Match Group, LLC. 3 | */ 4 | 5 | plugins { 6 | java 7 | kotlin("jvm") 8 | id("com.vanniktech.maven.publish") 9 | } 10 | 11 | dependencies { 12 | kotlinLibrary() 13 | implementation(Libs.snakeyaml) 14 | 15 | testImplementation(Libs.junit) 16 | } 17 | 18 | val generateVersionClass by tasks.register("generateVersionClass") { 19 | workingDir(project.rootDir) 20 | println("rootDir=${project.rootDir}") 21 | commandLine("./build-support/bin/write-version-class") 22 | } 23 | 24 | tasks.getByName("compileKotlin").dependsOn(generateVersionClass) 25 | -------------------------------------------------------------------------------- /WORKSPACE: -------------------------------------------------------------------------------- 1 | load("//bazel_support:repositories.bzl", "gitquery_dependencies") 2 | load("//bazel_support:constants.bzl", "MAVEN_ARTIFACTS") 3 | 4 | gitquery_dependencies() 5 | 6 | load("@io_bazel_rules_kotlin//kotlin:repositories.bzl", "kotlin_repositories") 7 | kotlin_repositories() 8 | 9 | load("@io_bazel_rules_kotlin//kotlin:core.bzl", "kt_register_toolchains") 10 | # kt_register_toolchains() 11 | register_toolchains("//:kotlin_toolchain") 12 | 13 | load("@rules_jvm_external//:defs.bzl", "maven_install") 14 | 15 | maven_install( 16 | artifacts = MAVEN_ARTIFACTS, 17 | repositories = [ 18 | "https://maven.google.com", 19 | "https://repo1.maven.org/maven2", 20 | ], 21 | ) 22 | -------------------------------------------------------------------------------- /gradle-plugin/src/main/kotlin/com/tinder/gitquery/GitQueryInitExtension.kt: -------------------------------------------------------------------------------- 1 | /* 2 | * © 2019 Match Group, LLC. 3 | */ 4 | 5 | package com.tinder.gitquery 6 | 7 | import com.tinder.gitquery.core.config.DEFAULT_FLAT_FILES 8 | import com.tinder.gitquery.core.config.DEFAULT_REVISION 9 | import com.tinder.gitquery.core.config.defaultIncludeGlobs 10 | import org.gradle.api.Project 11 | 12 | /** 13 | * Contains the settings for the gitquery initialize plugin that helps init and update a gitquery config file. 14 | */ 15 | open class GitQueryInitExtension(val project: Project) { 16 | var includeGlobs: List = defaultIncludeGlobs 17 | var excludeGlobs: List = defaultIncludeGlobs 18 | var flatFiles: Boolean = DEFAULT_FLAT_FILES 19 | var revision: String = DEFAULT_REVISION 20 | } 21 | -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | kotlin.code.style=official 2 | 3 | kapt.use.worker.api=true 4 | kapt.incremental.apt=true 5 | org.gradle.caching=true 6 | org.gradle.daemon=true 7 | org.gradle.parallel=true 8 | 9 | GROUP=com.tinder.gitquery 10 | VERSION_NAME=3.1.0 11 | 12 | POM_DESCRIPTION=A library for querying and syncing files in a remote git repo. 13 | POM_URL=https://github.com/Tinder/GitQuery 14 | POM_SCM_URL=https://github.com/Tinder/GitQuery 15 | POM_SCM_CONNECTION=scm:git:git://github.com/Tinder/GitQuery.git 16 | POM_SCM_DEV_CONNECTION=scm:git:ssh://git@github.com/Tinder/GitQuery.git 17 | POM_LICENCE_NAME=BSD 3-Clause "New" or "Revised" License 18 | POM_LICENCE_URL=https://opensource.org/licenses/BSD-3-Clause 19 | POM_LICENCE_DIST=repo 20 | POM_DEVELOPER_ID=Tinder 21 | POM_DEVELOPER_NAME=Tinder Open Source 22 | 23 | -------------------------------------------------------------------------------- /publish: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Deploy a jar, source jar, and javadoc jar to Sonatype's snapshot repo. 4 | # 5 | # Adapted from https://coderwall.com/p/9b_lfq and 6 | # http://benlimmer.com/2013/12/26/automatically-publish-javadoc-to-gh-pages-with-travis-ci/ 7 | 8 | REPO_URL="git@github.com:Tinder/GitQuery.git" 9 | BRANCH="master" 10 | 11 | set -e 12 | 13 | if [ "${CIRCLE_REPOSITORY_URL}" != "$REPO_URL" ]; then 14 | echo "Skipping snapshot deployment: wrong repository. Expected '$REPO_URL' but was '${CIRCLE_REPOSITORY_URL}'." 15 | elif [ "${CIRCLE_BRANCH}" != "$BRANCH" ]; then 16 | echo "Skipping snapshot deployment: wrong branch. Expected '$BRANCH' but was '${CIRCLE_BRANCH}'." 17 | else 18 | echo "Deploying snapshot..." 19 | ./gradlew clean uploadArchives --no-daemon --no-parallel 20 | echo "Snapshot deployed!" 21 | fi 22 | -------------------------------------------------------------------------------- /.github/workflows/gradle.yml: -------------------------------------------------------------------------------- 1 | name: Gradle 2 | on: 3 | push: 4 | branches: [ main ] 5 | pull_request: 6 | branches: [ main ] 7 | jobs: 8 | gradle: 9 | strategy: 10 | matrix: 11 | os: [ubuntu-latest] 12 | runs-on: ${{ matrix.os }} 13 | steps: 14 | - uses: actions/checkout@v2 15 | - uses: actions/setup-java@v1 16 | with: 17 | java-version: 11 18 | - name: Setup Gradle 19 | uses: gradle/gradle-build-action@v2 20 | 21 | - name: Detekt 22 | run: ./gradlew detekt 23 | - name: Lint 24 | run: ./gradlew ktlintCheck 25 | - name: Build 26 | run: ./gradlew test --stacktrace 27 | - name: Publish Unit Test Results 28 | uses: EnricoMi/publish-unit-test-result-action@v1 29 | if: always() 30 | with: 31 | files: "**/test-results/**/*.xml" 32 | -------------------------------------------------------------------------------- /.github/workflows/samples_gradle.yml: -------------------------------------------------------------------------------- 1 | name: Gradle Samples 2 | on: 3 | push: 4 | branches: [ main ] 5 | jobs: 6 | build: 7 | strategy: 8 | matrix: 9 | os: [ubuntu-latest] 10 | runs-on: ${{ matrix.os }} 11 | steps: 12 | - uses: actions/checkout@v2 13 | - uses: actions/setup-java@v1 14 | with: 15 | java-version: 11 16 | - uses: webfactory/ssh-agent@v0.5.4 17 | with: 18 | ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} 19 | - name: Setup Gradle 20 | uses: gradle/gradle-build-action@v2 21 | - name: Build Samples 22 | run: | 23 | for d in samples/*; do 24 | set -e 25 | [ -d "$d" ] || continue 26 | echo "== $d ==" 27 | pushd "$d" >/dev/null 28 | ../../gradlew gitQueryInit build 29 | popd >/dev/null 30 | done 31 | -------------------------------------------------------------------------------- /core/src/main/kotlin/com/tinder/gitquery/core/config/GitQueryInitConfig.kt: -------------------------------------------------------------------------------- 1 | /* 2 | * © 2019 Match Group, LLC. 3 | */ 4 | 5 | package com.tinder.gitquery.core.config 6 | 7 | /** 8 | * A model to encapsulate attributes related to init-config. 9 | */ 10 | data class GitQueryInitConfig( 11 | /* 12 | If true [default], when --init-config is used, the files attribute 13 | in the resulted saved config file will be a flat map of filename to revision values. 14 | If false, it will be a tree of directories as parent nodes and files as leaf nodes. 15 | */ 16 | var flatFiles: Boolean = DEFAULT_FLAT_FILES, 17 | /* A list of globs to include when generating the config file. */ 18 | var includeGlobs: List = defaultIncludeGlobs, 19 | /* A list of globs to exclude when generating the config file. */ 20 | var excludeGlobs: List = defaultExcludeGlobs, 21 | /* A default revision, if empty, HEAD is used. */ 22 | var revision: String = DEFAULT_REVISION 23 | ) 24 | -------------------------------------------------------------------------------- /gradle-plugin/src/main/kotlin/com/tinder/gitquery/GitQueryPlugin.kt: -------------------------------------------------------------------------------- 1 | /* 2 | * © 2019 Match Group, LLC. 3 | */ 4 | 5 | package com.tinder.gitquery 6 | 7 | import org.gradle.api.Plugin 8 | import org.gradle.api.Project 9 | 10 | /** 11 | * The main plugin class. 12 | */ 13 | class GitQueryPlugin : Plugin { 14 | override fun apply(project: Project) { 15 | val syncExtension = project.extensions.create("gitQuery", GitQuerySyncExtension::class.java, project) 16 | val initExtension = project.extensions.create("gitQueryInit", GitQueryInitExtension::class.java, project) 17 | 18 | project.tasks.register("gitQuery", GitQuerySyncTask::class.java, syncExtension) 19 | project.tasks.register("gitQueryInit", GitQueryInitTask::class.java, syncExtension, initExtension) 20 | 21 | project.afterEvaluate { 22 | if (syncExtension.autoSync) { 23 | project.tasks.getByName("assemble").dependsOn("gitQuery") 24 | } 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_kotlin//kotlin:kotlin.bzl", "kt_jvm_binary") 2 | 3 | java_library( 4 | name = "clikt", 5 | exports = [ 6 | "@maven//:com_github_ajalt_clikt_clikt_jvm", 7 | ], 8 | ) 9 | 10 | kt_jvm_binary( 11 | name = "GitQuery", 12 | deps = [ 13 | "//core:core", 14 | ":clikt", 15 | ], 16 | visibility = ["//visibility:public"], 17 | main_class = "com.tinder.gitquery.cli.GitQueryCliKt", 18 | srcs = [ 19 | "cli/src/main/kotlin/com/tinder/gitquery/cli/GitQueryCli.kt", 20 | ], 21 | ) 22 | 23 | load("@io_bazel_rules_kotlin//kotlin:core.bzl", "define_kt_toolchain") 24 | 25 | define_kt_toolchain( 26 | name = "kotlin_toolchain", 27 | api_version = "1.8", # "1.1", "1.2", "1.3", "1.4", "1.5" "1.6", "1.7", "1.8", or "1.9" 28 | jvm_target = "11", # "1.6", "1.8", "9", "10", "11", "12", "13", "15", "16", "17", "18", "19", "20" or "21" 29 | language_version = "1.8", # "1.1", "1.2", "1.3", "1.4", "1.5" "1.6", "1.7", "1.8", or "1.9" 30 | ) -------------------------------------------------------------------------------- /samples/protobuf-sync-code-gen/README.md: -------------------------------------------------------------------------------- 1 | ## Sync Protobuf and generate Java from it 2 | 3 | This example uses GitQuery to sync the addressbook example in the [protobuf repository](https://github.com/google/protobuf) and use [streem/pbandk](https://github.com/streem/pbandk) 4 | to generate Java. 5 | 6 | For this example the synced proto and generated code has been committed to show what it looks like. 7 | 8 | ### Building 9 | 10 | To build the code simply run: 11 | 12 | path/to/gradle installDist 13 | 14 | ### Running 15 | 16 | Once built, the start script will be at `build/install/addressbook/bin/addressbook` (and `.bat` version on Windows). 17 | There are two commands that both require filenames, `add-person` and `list-people`. To add a person to a file called 18 | `sample-book`, run: 19 | 20 | build/install/addressbook/bin/addressbook add-person sample-book 21 | 22 | Run multiple times to add multiple people. To see the list of people saved in `sample-book`, run: 23 | 24 | build/install/addressbook/bin/addressbook list-people sample-book -------------------------------------------------------------------------------- /gradle-plugin/src/main/kotlin/com/tinder/gitquery/GitQuerySyncExtension.kt: -------------------------------------------------------------------------------- 1 | /* 2 | * © 2019 Match Group, LLC. 3 | */ 4 | 5 | package com.tinder.gitquery 6 | 7 | import com.tinder.gitquery.core.config.DEFAULT_AUTO_SYNC 8 | import com.tinder.gitquery.core.config.DEFAULT_BRANCH 9 | import com.tinder.gitquery.core.config.DEFAULT_CLEAN_OUTPUT 10 | import com.tinder.gitquery.core.config.DEFAULT_CONFIG_FILENAME 11 | import com.tinder.gitquery.core.config.DEFAULT_GRADLE_REPO_DIR 12 | import com.tinder.gitquery.core.config.DEFAULT_OUTPUT_DIR 13 | import com.tinder.gitquery.core.config.DEFAULT_REMOTE 14 | import com.tinder.gitquery.core.config.DEFAULT_VERBOSE 15 | import org.gradle.api.Project 16 | 17 | /** 18 | * Contains the settings for our plugin. 19 | */ 20 | open class GitQuerySyncExtension(open val project: Project) { 21 | var autoSync: Boolean = DEFAULT_AUTO_SYNC 22 | var branch: String = DEFAULT_BRANCH 23 | var cleanOutput: Boolean = DEFAULT_CLEAN_OUTPUT 24 | var configFile: String = DEFAULT_CONFIG_FILENAME 25 | var outputDir: String = DEFAULT_OUTPUT_DIR 26 | var remote: String = DEFAULT_REMOTE 27 | var repoDir: String = DEFAULT_GRADLE_REPO_DIR 28 | var verbose: Boolean = DEFAULT_VERBOSE 29 | } 30 | -------------------------------------------------------------------------------- /core/src/main/kotlin/com/tinder/gitquery/core/utils/HashMapExtensions.kt: -------------------------------------------------------------------------------- 1 | package com.tinder.gitquery.core.utils 2 | 3 | @Suppress("UNCHECKED_CAST") 4 | internal fun HashMap.sortMap(): Map { 5 | return this.mapValues { 6 | if (it.value is HashMap<*, *>) { 7 | (it.value as HashMap).sortMap() 8 | } else { 9 | it.value 10 | } 11 | }.toSortedMap() 12 | } 13 | 14 | @Suppress("UNCHECKED_CAST") 15 | internal fun HashMap.insertNested(relativePath: String, revision: String) { 16 | val file = relativePath.substringAfterLast("/") 17 | val path = relativePath.substringBeforeLast("/") 18 | if (path == file) { 19 | this[file] = revision 20 | } else { 21 | val pathFirst = path.substringBefore("/") 22 | val pathFirstMap = if (this.containsKey(pathFirst) && this[pathFirst] is HashMap<*, *>) { 23 | this[pathFirst] as HashMap 24 | } else { 25 | val pathFirstMap = HashMap() 26 | this[pathFirst] = pathFirstMap 27 | pathFirstMap 28 | } 29 | pathFirstMap.insertNested(relativePath.substringAfter("/"), revision) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /gradle-plugin/build.gradle.kts: -------------------------------------------------------------------------------- 1 | /* 2 | * © 2019 Match Group, LLC. 3 | */ 4 | 5 | plugins { 6 | kotlin("jvm") 7 | `java-gradle-plugin` 8 | `java-library` 9 | id("com.gradle.plugin-publish") 10 | id("com.vanniktech.maven.publish") 11 | } 12 | 13 | val VERSION_NAME: String by project 14 | val GROUP: String by project 15 | 16 | gradlePlugin { 17 | plugins { 18 | create("GitQuery") { 19 | id = "com.tinder.gitquery" 20 | implementationClass = "com.tinder.gitquery.GitQueryPlugin" 21 | } 22 | } 23 | dependencies { 24 | implementation(project(":core")) 25 | testImplementation(Libs.junit) 26 | testImplementation(gradleTestKit()) 27 | } 28 | } 29 | 30 | pluginBundle { 31 | website = "https://github.com/Tinder/GitQuery" 32 | vcsUrl = "https://github.com/Tinder/GitQuery" 33 | description = "A Gradle plugin to query and sync files in a remote git repo." 34 | (plugins) { 35 | "GitQuery" { 36 | displayName = "Gradle Gitquery plugin" 37 | tags = listOf("gitquery", "protobuf", "git") 38 | version = VERSION_NAME 39 | } 40 | } 41 | mavenCoordinates { 42 | groupId = GROUP 43 | artifactId = "gradle-plugin" 44 | version = VERSION_NAME 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /samples/protobuf-sync-code-gen/src/main/proto/examples/addressbook.proto: -------------------------------------------------------------------------------- 1 | // See README.txt for information and build instructions. 2 | // 3 | // Note: START and END tags are used in comments to define sections used in 4 | // tutorials. They are not part of the syntax for Protocol Buffers. 5 | // 6 | // To get an in-depth walkthrough of this file and the related examples, see: 7 | // https://developers.google.com/protocol-buffers/docs/tutorials 8 | 9 | // [START declaration] 10 | syntax = "proto3"; 11 | package tutorial; 12 | 13 | import "google/protobuf/timestamp.proto"; 14 | // [END declaration] 15 | 16 | // [START java_declaration] 17 | option java_package = "com.example.tutorial"; 18 | option java_outer_classname = "AddressBookProtos"; 19 | // [END java_declaration] 20 | 21 | // [START csharp_declaration] 22 | option csharp_namespace = "Google.Protobuf.Examples.AddressBook"; 23 | // [END csharp_declaration] 24 | 25 | // [START messages] 26 | message Person { 27 | string name = 1; 28 | int32 id = 2; // Unique ID number for this person. 29 | string email = 3; 30 | 31 | enum PhoneType { 32 | MOBILE = 0; 33 | HOME = 1; 34 | WORK = 2; 35 | } 36 | 37 | message PhoneNumber { 38 | string number = 1; 39 | PhoneType type = 2; 40 | } 41 | 42 | repeated PhoneNumber phones = 4; 43 | 44 | google.protobuf.Timestamp last_updated = 5; 45 | } 46 | 47 | // Our address book file is just one of these. 48 | message AddressBook { 49 | repeated Person people = 1; 50 | } 51 | // [END messages] 52 | -------------------------------------------------------------------------------- /samples/protobuf-sync-groovy-dsl/src/main/proto/examples/addressbook.proto: -------------------------------------------------------------------------------- 1 | // See README.txt for information and build instructions. 2 | // 3 | // Note: START and END tags are used in comments to define sections used in 4 | // tutorials. They are not part of the syntax for Protocol Buffers. 5 | // 6 | // To get an in-depth walkthrough of this file and the related examples, see: 7 | // https://developers.google.com/protocol-buffers/docs/tutorials 8 | 9 | // [START declaration] 10 | syntax = "proto3"; 11 | package tutorial; 12 | 13 | import "google/protobuf/timestamp.proto"; 14 | // [END declaration] 15 | 16 | // [START java_declaration] 17 | option java_package = "com.example.tutorial"; 18 | option java_outer_classname = "AddressBookProtos"; 19 | // [END java_declaration] 20 | 21 | // [START csharp_declaration] 22 | option csharp_namespace = "Google.Protobuf.Examples.AddressBook"; 23 | // [END csharp_declaration] 24 | 25 | // [START messages] 26 | message Person { 27 | string name = 1; 28 | int32 id = 2; // Unique ID number for this person. 29 | string email = 3; 30 | 31 | enum PhoneType { 32 | MOBILE = 0; 33 | HOME = 1; 34 | WORK = 2; 35 | } 36 | 37 | message PhoneNumber { 38 | string number = 1; 39 | PhoneType type = 2; 40 | } 41 | 42 | repeated PhoneNumber phones = 4; 43 | 44 | google.protobuf.Timestamp last_updated = 5; 45 | } 46 | 47 | // Our address book file is just one of these. 48 | message AddressBook { 49 | repeated Person people = 1; 50 | } 51 | // [END messages] 52 | -------------------------------------------------------------------------------- /samples/protobuf-sync-code-gen/build.gradle.kts: -------------------------------------------------------------------------------- 1 | 2 | import com.google.protobuf.gradle.generateProtoTasks 3 | import com.google.protobuf.gradle.id 4 | import com.google.protobuf.gradle.ofSourceSet 5 | import com.google.protobuf.gradle.plugins 6 | import com.google.protobuf.gradle.protobuf 7 | import com.google.protobuf.gradle.protoc 8 | 9 | plugins { 10 | application 11 | id("com.google.protobuf") version "0.8.17" 12 | id("com.tinder.gitquery") version "3.0.12" 13 | java 14 | } 15 | 16 | val protoDir = "src/main/proto" 17 | 18 | gitQuery { 19 | autoSync = true 20 | configFile = "gitquery.yml" 21 | outputDir = protoDir 22 | repoDir = "tmp/.gitquery" 23 | } 24 | 25 | sourceSets { 26 | main { 27 | proto.srcDirs(protoDir) 28 | } 29 | } 30 | 31 | repositories { 32 | jcenter() 33 | if (System.getenv("CI") == "true") { 34 | mavenLocal() 35 | } 36 | maven("https://jitpack.io") 37 | } 38 | 39 | application { 40 | mainClass.set("com.examples.addressbook.MainKt") 41 | applicationName = "addressbook" 42 | } 43 | 44 | dependencies { 45 | implementation("com.google.protobuf:protobuf-java:3.19.1") 46 | } 47 | 48 | protobuf { 49 | generatedFilesBaseDir = "$projectDir/src" 50 | protoc { 51 | artifact = "com.google.protobuf:protoc:3.19.1" 52 | } 53 | generateProtoTasks { 54 | ofSourceSet("main").forEach { task -> 55 | if (task.name == "generateProto") { 56 | task.dependsOn(tasks.getByName("gitQuery")) 57 | } 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2021, Match Group, LLC 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | * Redistributions of source code must retain the above copyright 7 | notice, this list of conditions and the following disclaimer. 8 | * Redistributions in binary form must reproduce the above copyright 9 | notice, this list of conditions and the following disclaimer in the 10 | documentation and/or other materials provided with the distribution. 11 | * Neither the name of Match Group, LLC nor the names of its contributors 12 | may be used to endorse or promote products derived from this software 13 | without specific prior written permission. 14 | 15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 16 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 17 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 | DISCLAIMED. IN NO EVENT SHALL MATCH GROUP, LLC BE LIABLE FOR ANY 19 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 20 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 21 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 22 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 24 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | -------------------------------------------------------------------------------- /bazel_support/repositories.bzl: -------------------------------------------------------------------------------- 1 | """ 2 | Methods to assist in loading dependencies for GitQuery in WORKSPACE files 3 | """ 4 | 5 | load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") 6 | load( 7 | "//bazel_support:constants.bzl", 8 | "RULES_JVM_EXTERNAL_SHA", 9 | "RULES_JVM_EXTERNAL_TAG", 10 | "RULES_KOTLIN_SHA", 11 | "RULES_KOTLIN_VERSION", 12 | ) 13 | 14 | def _maybe(repo_rule, name, **kwargs): 15 | if not native.existing_rule(name): 16 | repo_rule(name = name, **kwargs) 17 | 18 | def gitquery_dependencies( 19 | rules_jvm_external_tag = RULES_JVM_EXTERNAL_TAG, 20 | rules_jvm_external_sha = RULES_JVM_EXTERNAL_SHA, 21 | rules_kotlin_version = RULES_KOTLIN_VERSION, 22 | rules_kotlin_sha = RULES_KOTLIN_SHA): 23 | _maybe( 24 | http_archive, 25 | name = "io_bazel_rules_kotlin", 26 | urls = ["https://github.com/bazelbuild/rules_kotlin/releases/download/%s/rules_kotlin-%s.tar.gz" % (RULES_KOTLIN_VERSION, RULES_KOTLIN_VERSION)], 27 | sha256 = RULES_KOTLIN_SHA, 28 | ) 29 | 30 | _maybe( 31 | http_archive, 32 | name = "rules_jvm_external", 33 | strip_prefix = "rules_jvm_external-%s" % rules_jvm_external_tag, 34 | sha256 = rules_jvm_external_sha, 35 | url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % rules_jvm_external_tag, 36 | ) 37 | 38 | _maybe( 39 | http_archive, 40 | name = "rules_java", 41 | sha256 = "eb7db63ed826567b2ceb1ec53d6b729e01636f72c9f5dfb6d2dfe55ad69d1d2a", 42 | url = "https://github.com/bazelbuild/rules_java/releases/download/7.2.0/rules_java-7.2.0.tar.gz", 43 | ) 44 | -------------------------------------------------------------------------------- /core/src/main/kotlin/com/tinder/gitquery/core/utils/PathUtils.kt: -------------------------------------------------------------------------------- 1 | package com.tinder.gitquery.core.utils 2 | 3 | /** 4 | * Checks if [path] is already absolute (starts with /), if yes return it, if not, return 5 | * [prefixPath]/[path]. [prefixPath] defaults to System.getProperty("user.dir"). 6 | * If the path is empty, a blank string is returned. 7 | */ 8 | fun toAbsolutePath( 9 | path: String, 10 | prefixPath: String = System.getProperty("user.dir"), 11 | ): String { 12 | return when { 13 | path.isBlank() -> { 14 | "" 15 | } 16 | path[0] == '/' || prefixPath.isBlank() -> { 17 | path 18 | } 19 | else -> { 20 | "$prefixPath/$path" 21 | } 22 | } 23 | } 24 | 25 | /** 26 | * Check for and create the output folder. Throws exceptions if there are issues. 27 | */ 28 | internal fun prepareOutputDirectory(outputPath: String, cleanOutput: Boolean, verbose: Boolean) { 29 | println("GitQuery: creating outputPath: $outputPath") 30 | 31 | // Either outputPath exists or we can create it 32 | check(0 == sh(verbose = verbose, "[ -d $outputPath ] || mkdir -p $outputPath")) { 33 | "OutputDir: $outputPath not found and couldn't be created" 34 | } 35 | 36 | // Either outputPath doesn't exist, or we can write to it. 37 | check(0 == sh(verbose = verbose, "[ ! -d $outputPath ] || [ -w $outputPath ]")) { 38 | "OutputDir: $outputPath does not have write permission" 39 | } 40 | 41 | if (cleanOutput) { 42 | // Clean the output folder so that if files were removed from the config, they don't get included. 43 | check(0 == sh(verbose = verbose, "(cd $outputPath && rm -rf *)")) { 44 | "Error cleaning outputDir with path: `$outputPath" 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /samples/sample1.yml: -------------------------------------------------------------------------------- 1 | # Describe a set of files to fetch from a given repository. 2 | --- 3 | schema: 4 | version: 1 5 | # remote - The remote repository to query files from. 6 | remote: git@github.com:protocolbuffers/protobuf.git 7 | # branch - The single branch that will be cloned on first run and pulled incrementally on 8 | # subsequent runs. The revision values used in [commits] and [files] must be available under [branch]. 9 | branch: master 10 | # A list of commit aliases that can be used in the `files` section. The value can be a revision: sha-1 or tag. 11 | commits: 12 | # https://github.com/protocolbuffers/protobuf/releases/tag/v3.12.2 13 | v3_12_2: v3.12.2 14 | # Specify a nested map of filenames to revision (or commit alias) included file that we 15 | # want to query and sync. The structure of `files` matches the directory structure of the 16 | # remote repo. A key whose value is a nested map is considered a directory. 17 | files: 18 | # This is a directory at the root of the repo pointed to by remote. 19 | src: 20 | google: 21 | protobuf: 22 | descriptor.proto: v3_12_2 23 | any.proto: v3_12_2 24 | api.proto: v3_12_2 25 | duration.proto: v3_12_2 26 | empty.proto: v3_12_2 27 | field_mask.proto: v3_12_2 28 | source_context.proto: v3_12_2 29 | struct.proto: v3_12_2 30 | timestamp.proto: v3_12_2 31 | type.proto: v3_12_2 32 | wrappers.proto: v3_12_2 33 | compiler: 34 | plugin.proto: v3_12_2 35 | # A directory to sync the queried files into. default: gitquery-output 36 | outputDir: gitquery-output 37 | # A directory to hold the intermediate cloned git repo. default: /tmp/gitquery/repo 38 | repoDir: /tmp/.gitquery 39 | # If true [default], cleans out the output folder prior to running sync. 40 | cleanOutput: false 41 | # A free form map of String -> Any - enabling self contained integration with various systems. 42 | extra: 43 | attr1: value1 44 | attr2: 1 -------------------------------------------------------------------------------- /core/src/main/kotlin/com/tinder/gitquery/core/utils/ShellUtils.kt: -------------------------------------------------------------------------------- 1 | package com.tinder.gitquery.core.utils 2 | 3 | /** 4 | * Run a shell command. 5 | */ 6 | internal fun sh(verbose: Boolean = false, vararg cmd: String): Int = 7 | runCommand(verbose = verbose, "sh", "-c", *cmd) 8 | 9 | internal fun runCommand(verbose: Boolean = false, vararg cmd: String): Int { 10 | val processBuilder = ProcessBuilder() 11 | .command(*cmd) 12 | .redirectErrorStream(true) 13 | 14 | if (verbose) { 15 | println(processBuilder.command().toString()) 16 | } 17 | 18 | val process = processBuilder.start() 19 | val lines = process.inputStream.bufferedReader().use { reader -> 20 | reader.readLines().joinToString(separator = "\n") 21 | } 22 | 23 | val exitCode = process.waitFor() 24 | if (verbose || (exitCode != 0 && lines.isNotEmpty())) { 25 | println(processBuilder.command().toString()) 26 | println("exit code = $exitCode") 27 | println(lines) 28 | } 29 | return exitCode 30 | } 31 | 32 | /** 33 | * Run a shell command for a value. 34 | */ 35 | internal fun shellResult(verbose: Boolean = false, vararg cmd: String): String = 36 | runCommandAndGetResult(verbose = verbose, "sh", "-c", *cmd) 37 | 38 | private fun runCommandAndGetResult(verbose: Boolean = false, vararg cmd: String): String { 39 | val processBuilder = ProcessBuilder() 40 | .command(*cmd) 41 | .redirectErrorStream(false) 42 | 43 | if (verbose) { 44 | println(processBuilder.command().toString()) 45 | } 46 | 47 | val process = processBuilder.start() 48 | val lines = process.inputStream.bufferedReader().use { reader -> 49 | reader.readLines().joinToString(separator = "\n") 50 | } 51 | 52 | val exitCode = process.waitFor() 53 | if ((exitCode != 0 && lines.isNotEmpty())) { 54 | println(processBuilder.command().toString()) 55 | println("exit code = $exitCode") 56 | println(lines) 57 | } 58 | return lines 59 | } 60 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Android Studio workspace files 2 | /.idea 3 | !/.idea/copyright 4 | *.iml 5 | workspace.xml 6 | 7 | # Ruby / Danger files 8 | vendor/ 9 | 10 | # built application files 11 | *.ap_ 12 | 13 | # files for the dex VM 14 | *.dex 15 | 16 | # Java class files 17 | *.class 18 | 19 | # Mac files. 20 | .DS_Store 21 | 22 | # generated files 23 | out 24 | gen/ 25 | captures/ 26 | fastlane/* 27 | scripts/changelog* 28 | 29 | # Local configuration file (sdk path, etc) 30 | local.properties 31 | 32 | # Eclipse project files 33 | .classpath 34 | .project 35 | encodings.xml 36 | vcs.xml 37 | 38 | build 39 | !BUILD 40 | 41 | ### Android template 42 | # Built application files 43 | *.apk 44 | 45 | # Generated files 46 | out/ 47 | 48 | # Gradle files 49 | .gradle/ 50 | build/ 51 | 52 | # Local configuration file (sdk path, etc) 53 | 54 | # Proguard folder generated by Eclipse 55 | proguard/ 56 | 57 | # Log Files 58 | *.log 59 | 60 | # Android Studio Navigation editor temp files 61 | .navigation/ 62 | 63 | ### JetBrains template 64 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm 65 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 66 | 67 | ## File-based project format: 68 | *.iws 69 | 70 | ## Plugin-specific files: 71 | 72 | # IntelliJ 73 | .idea 74 | .idea/workspace.xml 75 | .idea/tasks.xml 76 | .idea/gradle.xml 77 | .idea/dictionaries 78 | .idea/libraries 79 | .settings 80 | 81 | # mpeltonen/sbt-idea plugin 82 | .idea_modules/ 83 | 84 | # JIRA plugin 85 | atlassian-ide-plugin.xml 86 | 87 | # Crashlytics plugin (for Android Studio and IntelliJ) 88 | com_crashlytics_export_strings.xml 89 | crashlytics.properties 90 | crashlytics-build.properties 91 | fabric.properties 92 | 93 | jrebel.properties 94 | 95 | # Bundler 96 | .bundle/ 97 | 98 | # Ndk 99 | obj/ 100 | 101 | gitquery-output 102 | 103 | tmp 104 | 105 | samples/sample1-generate-nested.yml 106 | 107 | # Bazel 108 | bazel-* 109 | -------------------------------------------------------------------------------- /core/src/main/kotlin/com/tinder/gitquery/core/utils/RepoUtils.kt: -------------------------------------------------------------------------------- 1 | /* 2 | * © 2019 Match Group, LLC. 3 | */ 4 | 5 | package com.tinder.gitquery.core.utils 6 | 7 | /** 8 | * Utility methods related to a Git repository. 9 | */ 10 | 11 | /** 12 | * Clone or pull the `remote` repo @ `branch` into file(`repoDir`) 13 | * 14 | * @param remote the url to the remote git repo 15 | * @param branch a branch in the remote repo 16 | * @param repoDir where the remote repo(s) can be cloned locally and stored temporarily. 17 | */ 18 | internal fun prepareRepo(remote: String, branch: String, repoDir: String, verbose: Boolean) { 19 | val repoExists = repoExists(repoDir, verbose = verbose) 20 | var exitCode = 0 21 | 22 | // If repo directory already exists, it means we have already cloned the remote repo 23 | if (repoExists) { 24 | // Since we have th repo already, fetch the right branch from origin and checkout the branch 25 | // In cases where the branch changes, `git checkout $branch` will fail silently, prompting 26 | // us to do a clean single branch clone of the repository. 27 | exitCode = sh( 28 | verbose, 29 | "cd $repoDir && git checkout $branch &>/dev/null && git pull origin $branch --tags" 30 | ) 31 | } 32 | 33 | // Either: 34 | // 1) repoDir doesn't exist 35 | // 2) or, we couldn't pull the branch that we wanted. 36 | // Cleanup and try a fresh clone. 37 | if (exitCode != 0 || !repoExists) { 38 | sh(verbose, "rm -rf $repoDir") 39 | exitCode = sh(verbose, "git clone --single-branch -b $branch $remote $repoDir") 40 | } 41 | 42 | check(exitCode == 0) { "Error cloning/updating repo $remote into directory $repoDir" } 43 | } 44 | 45 | /** 46 | * Get current revision of the repoDir. 47 | */ 48 | internal fun repoHeadRevision(repoDir: String, verbose: Boolean): String { 49 | return shellResult(verbose, "(cd $repoDir && git rev-parse HEAD)") 50 | } 51 | 52 | /** 53 | * Checks the existence of the repoDir 54 | */ 55 | internal fun repoExists(repoDir: String, verbose: Boolean): Boolean { 56 | return 0 == sh(verbose, "[ -d $repoDir ] && [ -d $repoDir/.git ]") 57 | } 58 | -------------------------------------------------------------------------------- /samples/sample2-generate-nested.yml: -------------------------------------------------------------------------------- 1 | !!com.tinder.gitquery.core.config.GitQueryConfig 2 | branch: master 3 | cleanOutput: true 4 | commits: &id001 {} 5 | extra: *id001 6 | files: 7 | README.md: v3.14.0 8 | benchmarks: 9 | datasets: 10 | google_message1: 11 | proto3: 12 | benchmark_message1_proto3.proto: v3.14.0 13 | google_message2: 14 | benchmark_message2.proto: v3.14.0 15 | google_message3: 16 | benchmark_message3.proto: v3.14.0 17 | benchmark_message3_1.proto: v3.14.0 18 | benchmark_message3_2.proto: v3.14.0 19 | benchmark_message3_3.proto: v3.14.0 20 | benchmark_message3_4.proto: v3.14.0 21 | benchmark_message3_5.proto: v3.14.0 22 | benchmark_message3_6.proto: v3.14.0 23 | benchmark_message3_7.proto: v3.14.0 24 | benchmark_message3_8.proto: v3.14.0 25 | google_message4: 26 | benchmark_message4.proto: v3.14.0 27 | benchmark_message4_1.proto: v3.14.0 28 | benchmark_message4_2.proto: v3.14.0 29 | benchmark_message4_3.proto: v3.14.0 30 | src: 31 | google: 32 | protobuf: 33 | compiler: 34 | cpp: 35 | cpp_test_bad_identifiers.proto: v3.14.0 36 | cpp_test_large_enum_value.proto: v3.14.0 37 | plugin.proto: v3.14.0 38 | util: 39 | internal: 40 | testdata: 41 | anys.proto: v3.14.0 42 | books.proto: v3.14.0 43 | default_value.proto: v3.14.0 44 | default_value_test.proto: v3.14.0 45 | field_mask.proto: v3.14.0 46 | maps.proto: v3.14.0 47 | oneofs.proto: v3.14.0 48 | proto3.proto: v3.14.0 49 | struct.proto: v3.14.0 50 | timestamp_duration.proto: v3.14.0 51 | wrappers.proto: v3.14.0 52 | json_format.proto: v3.14.0 53 | json_format_proto3.proto: v3.14.0 54 | message_differencer_unittest.proto: v3.14.0 55 | initConfig: 56 | excludeGlobs: 57 | - '**/ruby/**' 58 | - '**/proto2/**' 59 | flatFiles: false 60 | includeGlobs: 61 | - '**/src/google/protobuf/**/*.proto' 62 | - '**/benchmarks/**/*.proto' 63 | - README.md 64 | revision: v3.14.0 65 | outputDir: gitquery-output 66 | remote: git@github.com:protocolbuffers/protobuf.git 67 | repoDir: /tmp/gitquery/repo 68 | schema: 69 | version: '' 70 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | set DIRNAME=%~dp0 12 | if "%DIRNAME%" == "" set DIRNAME=. 13 | set APP_BASE_NAME=%~n0 14 | set APP_HOME=%DIRNAME% 15 | 16 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 17 | set DEFAULT_JVM_OPTS="-Xmx64m" 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windows variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | 53 | :win9xME_args 54 | @rem Slurp the command line arguments. 55 | set CMD_LINE_ARGS= 56 | set _SKIP=2 57 | 58 | :win9xME_args_slurp 59 | if "x%~1" == "x" goto execute 60 | 61 | set CMD_LINE_ARGS=%* 62 | 63 | :execute 64 | @rem Setup the command line 65 | 66 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 67 | 68 | @rem Execute Gradle 69 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 70 | 71 | :end 72 | @rem End local scope for the variables with windows NT shell 73 | if "%ERRORLEVEL%"=="0" goto mainEnd 74 | 75 | :fail 76 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 77 | rem the _cmd.exe /c_ return code! 78 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 79 | exit /b 1 80 | 81 | :mainEnd 82 | if "%OS%"=="Windows_NT" endlocal 83 | 84 | :omega 85 | -------------------------------------------------------------------------------- /gradle-plugin/src/main/kotlin/com/tinder/gitquery/GitQueryInitTask.kt: -------------------------------------------------------------------------------- 1 | /* 2 | * © 2019 Match Group, LLC. 3 | */ 4 | 5 | package com.tinder.gitquery 6 | 7 | import com.tinder.gitquery.core.GitQueryInit.initConfig 8 | import org.gradle.api.tasks.Input 9 | import org.gradle.api.tasks.TaskAction 10 | import javax.inject.Inject 11 | 12 | /** 13 | * A Gradle task that given a yaml file, describing a set of files in a remote git repo, and an 14 | * intermediate (repo) directory, query, fetch and sync those files into a given output directory. 15 | * 16 | * For more details see the README.md file in the root of this project. 17 | * 18 | * @param syncExtension [GitQuerySyncExtension] 19 | */ 20 | open class GitQueryInitTask @Inject constructor( 21 | syncExtension: GitQuerySyncExtension, 22 | initExtension: GitQueryInitExtension 23 | ) : GitQuerySyncTask(syncExtension = syncExtension) { 24 | 25 | /* A list of globs to include when generating the config file. */ 26 | @Input 27 | val includeGlobs: List = initExtension.includeGlobs 28 | 29 | /* A list of globs to exclude when generating the config file. */ 30 | @Input 31 | val excludeGlobs: List = initExtension.excludeGlobs 32 | 33 | /* 34 | * If true (default), when --init-config is used, the files attribute 35 | * in the resulted saved config file will be a flat map of filename to sha values. 36 | * If false, it will be a tree of directories as parent nodes and files as leaf nodes. 37 | */ 38 | @Input 39 | val flatFiles: Boolean = initExtension.flatFiles 40 | 41 | /* 42 | * A revision under [branch], when --init-config is applied, that will be used as the default revision for files. 43 | */ 44 | @Input 45 | val revision: String = initExtension.revision 46 | 47 | init { 48 | group = "build" 49 | description = "Initialize or update the files in the config file based on includeGlobs, and excludeGlobs" 50 | } 51 | 52 | @TaskAction 53 | override fun taskAction() { 54 | initConfig( 55 | configFile = configFileName(), 56 | config = readConfig(createIfNotExists = true).apply { 57 | if (includeGlobs.isNotEmpty()) { 58 | initConfig.includeGlobs = includeGlobs 59 | } 60 | if (excludeGlobs.isNotEmpty()) { 61 | initConfig.excludeGlobs = excludeGlobs 62 | } 63 | if (revision.isNotBlank()) { 64 | initConfig.revision = revision 65 | } 66 | initConfig.flatFiles = flatFiles 67 | }, 68 | verbose = verbose, 69 | buildDir = project.buildDir.path 70 | ) 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /core/src/test/kotlin/com/tinder/gitquery/core/GitQueryConfigTest.kt: -------------------------------------------------------------------------------- 1 | package com.tinder.gitquery.core 2 | 3 | import com.tinder.gitquery.core.config.GitQueryConfig 4 | import org.junit.After 5 | import org.junit.Before 6 | import org.junit.Rule 7 | import org.junit.Test 8 | import org.junit.rules.TemporaryFolder 9 | import java.lang.IllegalArgumentException 10 | import java.lang.IllegalStateException 11 | 12 | class GitQueryConfigTest { 13 | @get:Rule 14 | val testProjectDir = TemporaryFolder() 15 | 16 | @Before 17 | fun setup() { 18 | testProjectDir.create() 19 | testProjectDir.apply { 20 | newFile("gitquery.yml").appendText(getContentConfig()) 21 | } 22 | } 23 | 24 | @After 25 | fun tearDown() { 26 | testProjectDir.delete() 27 | } 28 | 29 | @Test 30 | fun `loadConfig - given file exists, should return config`() { 31 | // When 32 | val config = GitQueryConfig.load("${testProjectDir.root}/gitquery.yml", false) 33 | 34 | // Then 35 | assert(config.remote == "https://github.com/aminghadersohi/ProtoExample.git") 36 | assert(config.branch == "master") 37 | assert(config.commits == mapOf("latest" to "d654b510d2689e8ee56d23d03dff2be742737f86")) 38 | assert( 39 | config.files == mapOf( 40 | "README.md" to "latest", 41 | "definitions" to mapOf("user.proto" to "42933446d0321958e8c12216d04b9f0c382ebf1b") 42 | ) 43 | ) 44 | } 45 | 46 | @Test 47 | fun `loadConfig - given blank filename, show throw`() { 48 | // Given 49 | val filename = " " 50 | 51 | // When 52 | val actualError = kotlin.runCatching { 53 | GitQueryConfig.load(filename, false) 54 | }.exceptionOrNull() 55 | 56 | require(actualError is IllegalArgumentException) 57 | assert( 58 | actualError.message == "Input filename may not be a blank string ($filename)" 59 | ) 60 | } 61 | 62 | @Test 63 | fun `loadConfig - given non existent filename, show throw`() { 64 | // Given 65 | val filename = "config.yaml" 66 | 67 | // When 68 | val actualError = kotlin.runCatching { 69 | GitQueryConfig.load(filename, false) 70 | }.exceptionOrNull() 71 | 72 | require(actualError is IllegalStateException) 73 | assert(actualError.message == "Config file does not exist ($filename)") 74 | } 75 | } 76 | 77 | private fun getContentConfig() = 78 | """ 79 | --- 80 | schema: 81 | version: 1 82 | remote: https://github.com/aminghadersohi/ProtoExample.git 83 | branch: master 84 | commits: 85 | latest: d654b510d2689e8ee56d23d03dff2be742737f86 86 | files: 87 | README.md: latest 88 | definitions: 89 | user.proto: 42933446d0321958e8c12216d04b9f0c382ebf1b 90 | 91 | """.trimIndent() 92 | -------------------------------------------------------------------------------- /samples/sample2-generate-flat.yml: -------------------------------------------------------------------------------- 1 | !!com.tinder.gitquery.core.config.GitQueryConfig 2 | branch: master 3 | cleanOutput: true 4 | commits: &id001 {} 5 | extra: *id001 6 | files: 7 | README.md: v3.14.0 8 | benchmarks/datasets/google_message1/proto3/benchmark_message1_proto3.proto: v3.14.0 9 | benchmarks/datasets/google_message2/benchmark_message2.proto: v3.14.0 10 | benchmarks/datasets/google_message3/benchmark_message3.proto: v3.14.0 11 | benchmarks/datasets/google_message3/benchmark_message3_1.proto: v3.14.0 12 | benchmarks/datasets/google_message3/benchmark_message3_2.proto: v3.14.0 13 | benchmarks/datasets/google_message3/benchmark_message3_3.proto: v3.14.0 14 | benchmarks/datasets/google_message3/benchmark_message3_4.proto: v3.14.0 15 | benchmarks/datasets/google_message3/benchmark_message3_5.proto: v3.14.0 16 | benchmarks/datasets/google_message3/benchmark_message3_6.proto: v3.14.0 17 | benchmarks/datasets/google_message3/benchmark_message3_7.proto: v3.14.0 18 | benchmarks/datasets/google_message3/benchmark_message3_8.proto: v3.14.0 19 | benchmarks/datasets/google_message4/benchmark_message4.proto: v3.14.0 20 | benchmarks/datasets/google_message4/benchmark_message4_1.proto: v3.14.0 21 | benchmarks/datasets/google_message4/benchmark_message4_2.proto: v3.14.0 22 | benchmarks/datasets/google_message4/benchmark_message4_3.proto: v3.14.0 23 | src/google/protobuf/compiler/cpp/cpp_test_bad_identifiers.proto: v3.14.0 24 | src/google/protobuf/compiler/cpp/cpp_test_large_enum_value.proto: v3.14.0 25 | src/google/protobuf/compiler/plugin.proto: v3.14.0 26 | src/google/protobuf/util/internal/testdata/anys.proto: v3.14.0 27 | src/google/protobuf/util/internal/testdata/books.proto: v3.14.0 28 | src/google/protobuf/util/internal/testdata/default_value.proto: v3.14.0 29 | src/google/protobuf/util/internal/testdata/default_value_test.proto: v3.14.0 30 | src/google/protobuf/util/internal/testdata/field_mask.proto: v3.14.0 31 | src/google/protobuf/util/internal/testdata/maps.proto: v3.14.0 32 | src/google/protobuf/util/internal/testdata/oneofs.proto: v3.14.0 33 | src/google/protobuf/util/internal/testdata/proto3.proto: v3.14.0 34 | src/google/protobuf/util/internal/testdata/struct.proto: v3.14.0 35 | src/google/protobuf/util/internal/testdata/timestamp_duration.proto: v3.14.0 36 | src/google/protobuf/util/internal/testdata/wrappers.proto: v3.14.0 37 | src/google/protobuf/util/json_format.proto: v3.14.0 38 | src/google/protobuf/util/json_format_proto3.proto: v3.14.0 39 | src/google/protobuf/util/message_differencer_unittest.proto: v3.14.0 40 | initConfig: 41 | excludeGlobs: 42 | - '**/ruby/**' 43 | - '**/proto2/**' 44 | flatFiles: true 45 | includeGlobs: 46 | - '**/src/google/protobuf/**/*.proto' 47 | - '**/benchmarks/**/*.proto' 48 | - README.md 49 | revision: v3.14.0 50 | outputDir: gitquery-output 51 | remote: git@github.com:protocolbuffers/protobuf.git 52 | repoDir: /tmp/gitquery/repo 53 | schema: 54 | version: '' 55 | -------------------------------------------------------------------------------- /gradle-plugin/src/test/kotlin/com/tinder/gitquery/GitQueryInitTaskTest.kt: -------------------------------------------------------------------------------- 1 | /* 2 | * © 2019 Match Group, LLC. 3 | */ 4 | 5 | package com.tinder.gitquery 6 | 7 | import org.gradle.testkit.runner.GradleRunner 8 | import org.gradle.testkit.runner.TaskOutcome 9 | import org.junit.After 10 | import org.junit.Before 11 | import org.junit.Rule 12 | import org.junit.Test 13 | import org.junit.rules.TemporaryFolder 14 | import java.io.File 15 | 16 | class GitQueryInitTaskTest { 17 | 18 | @get:Rule 19 | val testProjectDir = TemporaryFolder() 20 | 21 | @Before 22 | fun setup() { 23 | testProjectDir.create() 24 | } 25 | 26 | @After 27 | fun tearDown() { 28 | testProjectDir.delete() 29 | } 30 | 31 | @Test 32 | fun taskGitQueryInitShouldCreateGitQueryConfig() { 33 | testProjectDir.apply { 34 | newFile("build.gradle").appendText(getBuildGradleSetup()) 35 | } 36 | 37 | val result = GradleRunner.create() 38 | .withProjectDir(testProjectDir.root) 39 | .withArguments("gitQueryInit") 40 | .withPluginClasspath() 41 | .build() 42 | 43 | assert(result.task(":gitQueryInit")?.outcome == TaskOutcome.SUCCESS) 44 | assert(result.output.contains("GitQuery: init complete")) 45 | assert(File("${testProjectDir.root}/gitquery.yml").exists()) 46 | } 47 | 48 | @Test 49 | fun taskGitQueryInitShouldUpdateGitQueryConfig() { 50 | testProjectDir.apply { 51 | newFile("build.gradle").appendText(getBuildGradleSetup()) 52 | newFile("gitquery.yml").appendText(getContentConfig()) 53 | } 54 | 55 | val result = GradleRunner.create() 56 | .withProjectDir(testProjectDir.root) 57 | .withArguments("gitQueryInit") 58 | .withPluginClasspath() 59 | .build() 60 | 61 | assert(result.task(":gitQueryInit")?.outcome == TaskOutcome.SUCCESS) 62 | assert(result.output.contains("GitQuery: init complete")) 63 | assert(File("${testProjectDir.root}/gitquery.yml").exists()) 64 | } 65 | 66 | private fun getBuildGradleSetup() = 67 | """ 68 | plugins { 69 | id 'com.tinder.gitquery' 70 | } 71 | 72 | gitQuery { 73 | configFile = "gitquery.yml" 74 | repoDir = "tmp/remote" 75 | branch = "master" 76 | remote = "https://github.com/aminghadersohi/ProtoExample.git" 77 | } 78 | 79 | gitQueryInit { 80 | includeGlobs = ["**/*.proto"] 81 | } 82 | """.trimIndent() 83 | 84 | private fun getContentConfig() = 85 | """ 86 | --- 87 | schema: 88 | version: 1 89 | remote: https://github.com/aminghadersohi/ProtoExample.git 90 | branch: master 91 | commits: 92 | latest: d654b510d2689e8ee56d23d03dff2be742737f86 93 | files: 94 | README.md: latest 95 | definitions: 96 | user.proto: 42933446d0321958e8c12216d04b9f0c382ebf1b 97 | 98 | """.trimIndent() 99 | } 100 | -------------------------------------------------------------------------------- /gradle-plugin/src/main/kotlin/com/tinder/gitquery/GitQuerySyncTask.kt: -------------------------------------------------------------------------------- 1 | /* 2 | * © 2019 Match Group, LLC. 3 | */ 4 | 5 | package com.tinder.gitquery 6 | 7 | import com.tinder.gitquery.core.GitQuerySync 8 | import com.tinder.gitquery.core.config.GitQueryConfig 9 | import com.tinder.gitquery.core.utils.toAbsolutePath 10 | import org.gradle.api.DefaultTask 11 | import org.gradle.api.tasks.Input 12 | import org.gradle.api.tasks.TaskAction 13 | import javax.inject.Inject 14 | 15 | /** 16 | * A Gradle task that given a yaml file, describing a set of files in a remote git repo, and an 17 | * intermediate (repo) directory, query, fetch and sync those files into a given output directory. 18 | * 19 | * For more details see the README.md file in the root of this project. 20 | * 21 | * @param syncExtension [GitQuerySyncExtension] 22 | */ 23 | open class GitQuerySyncTask @Inject constructor( 24 | syncExtension: GitQuerySyncExtension 25 | ) : DefaultTask() { 26 | 27 | /** 28 | * The relative (to projectDir) path to a yaml file that describe a set of files to fetch/sync from a given 29 | * repository. 30 | */ 31 | @Input 32 | val configFile: String = syncExtension.configFile 33 | 34 | /** The remote repository to query files from. */ 35 | @Input 36 | val remote: String = syncExtension.remote 37 | 38 | /** 39 | * The single branch that will be cloned on first run and pulled incrementally on subsequent 40 | * runs. The revision values used in [commits] and [files] must exist under [branch]. 41 | */ 42 | @Input 43 | val branch: String = syncExtension.branch 44 | 45 | /** A directory to hold the intermediate cloned git repo. */ 46 | @Input 47 | val repoDir: String = syncExtension.repoDir 48 | 49 | /** A directory to sync the queried files into. */ 50 | @Input 51 | val outputDir: String = syncExtension.outputDir 52 | 53 | /** If true (default), cleans out the output folder prior to running sync. */ 54 | @Input 55 | val cleanOutput: Boolean = syncExtension.cleanOutput 56 | 57 | /** An boolean to enable showing the underlying commands and their outputs in the console. (default: false) */ 58 | @Input 59 | val verbose: Boolean = syncExtension.verbose 60 | 61 | init { 62 | group = "build" 63 | description = "Sync the files listed in the config file" 64 | } 65 | 66 | @TaskAction 67 | open fun taskAction() { 68 | GitQuerySync.sync( 69 | config = readConfig(createIfNotExists = false), 70 | verbose = verbose, 71 | projectDir = project.projectDir.path, 72 | buildDir = project.buildDir.path 73 | ) 74 | } 75 | 76 | protected fun readConfig(createIfNotExists: Boolean): GitQueryConfig { 77 | return GitQueryConfig.load( 78 | filename = configFileName(), 79 | createIfNotExists = createIfNotExists 80 | ).also { 81 | if (remote.isNotBlank()) { 82 | it.remote = remote 83 | } 84 | if (branch.isNotBlank()) { 85 | it.branch = branch 86 | } 87 | if (repoDir.isNotBlank()) { 88 | it.repoDir = repoDir 89 | } 90 | if (outputDir.isNotBlank()) { 91 | it.outputDir = outputDir 92 | } 93 | it.cleanOutput = cleanOutput 94 | } 95 | } 96 | 97 | protected fun configFileName(): String { 98 | return toAbsolutePath( 99 | path = configFile, 100 | prefixPath = "${project.projectDir}" 101 | ) 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /gradle-plugin/src/test/kotlin/com/tinder/gitquery/GitQuerySyncTaskTest.kt: -------------------------------------------------------------------------------- 1 | /* 2 | * © 2019 Match Group, LLC. 3 | */ 4 | 5 | package com.tinder.gitquery 6 | 7 | import org.gradle.testkit.runner.GradleRunner 8 | import org.gradle.testkit.runner.TaskOutcome 9 | import org.junit.After 10 | import org.junit.Before 11 | import org.junit.Rule 12 | import org.junit.Test 13 | import org.junit.rules.TemporaryFolder 14 | import java.io.File 15 | 16 | class GitQuerySyncTaskTest { 17 | 18 | @get:Rule 19 | val testProjectDir = TemporaryFolder() 20 | 21 | @Before 22 | fun setup() { 23 | testProjectDir.create() 24 | } 25 | 26 | @After 27 | fun tearDown() { 28 | testProjectDir.delete() 29 | } 30 | 31 | @Test 32 | fun taskCreateFolderWithFilesAtRoot() { 33 | testProjectDir.apply { 34 | newFolder("gitquery-output") 35 | newFile("build.gradle").appendText(getBuildGradleSetup()) 36 | newFile("gitquery.yml").appendText(getContentConfig()) 37 | } 38 | 39 | val result = GradleRunner.create() 40 | .withProjectDir(testProjectDir.root) 41 | .withArguments("gitQuery") 42 | .withPluginClasspath() 43 | .build() 44 | 45 | assert(result.task(":gitQuery")?.outcome == TaskOutcome.SUCCESS) 46 | assert(result.output.contains("GitQuery: creating outputPath")) 47 | assert(File("${testProjectDir.root}/gitquery-output/definitions/user.proto").exists()) 48 | } 49 | 50 | @Test 51 | fun missingRemoteInConfigFileIsThrowingException() { 52 | testProjectDir.apply { 53 | newFolder("gitquery-output") 54 | newFile("build.gradle").appendText(getBuildGradleSetup()) 55 | newFile("gitquery.yml").appendText(getContentMissingRemoteConfig()) 56 | } 57 | 58 | val result = GradleRunner.create() 59 | .withProjectDir(testProjectDir.root) 60 | .withArguments("gitQuery") 61 | .withPluginClasspath() 62 | .buildAndFail() 63 | 64 | println(result.output) 65 | assert(result.task(":gitQuery")?.outcome == TaskOutcome.FAILED) 66 | assert(result.output.contains("Parameter remote may not be a blank string ()")) 67 | } 68 | 69 | @Test 70 | fun wrongConfigFormatThrowingException() { 71 | testProjectDir.apply { 72 | newFolder("gitquery-output") 73 | newFile("build.gradle").appendText(getBuildGradleSetup()) 74 | newFile("gitquery.yml").appendText(getContentWrongFormat()) 75 | } 76 | 77 | val result = GradleRunner.create() 78 | .withProjectDir(testProjectDir.root) 79 | .withArguments("gitQuery") 80 | .withPluginClasspath() 81 | .buildAndFail() 82 | 83 | assert(result.output.contains("Can't construct a java object for tag:yaml.org")) 84 | assert(result.task(":gitQuery")?.outcome == TaskOutcome.FAILED) 85 | } 86 | 87 | private fun getBuildGradleSetup() = 88 | """ 89 | plugins { 90 | id 'com.tinder.gitquery' 91 | } 92 | 93 | gitQuery { 94 | configFile = "gitquery.yml" 95 | outputDir = "gitquery-output" 96 | repoDir = "tmp/remote" 97 | } 98 | """.trimIndent() 99 | 100 | private fun getContentConfig() = 101 | """ 102 | --- 103 | schema: 104 | version: 1 105 | remote: https://github.com/aminghadersohi/ProtoExample.git 106 | branch: master 107 | commits: 108 | latest: d654b510d2689e8ee56d23d03dff2be742737f86 109 | files: 110 | README.md: latest 111 | definitions: 112 | user.proto: 42933446d0321958e8c12216d04b9f0c382ebf1b 113 | 114 | """.trimIndent() 115 | 116 | private fun getContentMissingRemoteConfig() = 117 | """ 118 | --- 119 | schema: 120 | version: 1 121 | branch: master 122 | files: 123 | definitions: 124 | user.proto: 42933446d0321958e8c12216d04b9f0c382ebf1b 125 | """.trimIndent() 126 | 127 | private fun getContentWrongFormat() = 128 | """--- 129 | master{ } 130 | incorrectFormat 131 | """.trimIndent() 132 | } 133 | -------------------------------------------------------------------------------- /core/src/main/kotlin/com/tinder/gitquery/core/GitQuerySync.kt: -------------------------------------------------------------------------------- 1 | /* 2 | * © 2019 Match Group, LLC. 3 | */ 4 | 5 | package com.tinder.gitquery.core 6 | 7 | import com.tinder.gitquery.core.config.GitQueryConfig 8 | import com.tinder.gitquery.core.utils.prepareOutputDirectory 9 | import com.tinder.gitquery.core.utils.prepareRepo 10 | import com.tinder.gitquery.core.utils.sh 11 | import com.tinder.gitquery.core.utils.toAbsolutePath 12 | 13 | /** 14 | * A utility class that given a yaml file, describing a set of files in a remote git repo, and an 15 | * intermediate (repo) directory, query, fetch and sync those files into a given output directory. 16 | * 17 | * For more details see the README.md file in the root of this project. 18 | */ 19 | object GitQuerySync { 20 | 21 | /** 22 | * Sync all files 23 | * 24 | * @param config a yaml file that describe a set of files to fetch/sync from a given repository 25 | * @param verbose if true, it will print its operations to standard out. 26 | */ 27 | fun sync( 28 | config: GitQueryConfig, 29 | verbose: Boolean = false, 30 | projectDir: String = System.getProperty("user.dir"), 31 | buildDir: String = System.getProperty("user.dir") + "/build" 32 | ) { 33 | config.validate() 34 | 35 | val outputPath = toAbsolutePath( 36 | path = config.outputDir, 37 | prefixPath = projectDir 38 | ) 39 | 40 | val actualRepoPath = config.getActualRepoPath(buildDir) 41 | 42 | prepareRepo(config.remote, config.branch, actualRepoPath, verbose = verbose) 43 | 44 | prepareOutputDirectory( 45 | outputPath = outputPath, 46 | cleanOutput = config.cleanOutput, 47 | verbose = verbose 48 | ) 49 | 50 | // Sync all the files in `config.files`, recursively. 51 | syncFiles( 52 | fileMap = config.files, 53 | remote = config.remote, 54 | commits = config.commits, 55 | repoPath = actualRepoPath, 56 | outputPath = outputPath, 57 | relativePath = "", 58 | verbose = verbose 59 | ) 60 | } 61 | 62 | /** 63 | * Sync all files recursively. 64 | */ 65 | @Suppress("LongParameterList") 66 | private fun syncFiles( 67 | fileMap: Map, 68 | commits: Map, 69 | remote: String, 70 | repoPath: String, 71 | outputPath: String, 72 | relativePath: String, 73 | verbose: Boolean, 74 | ) { 75 | fileMap.forEach { (filename, value) -> 76 | val path = if (relativePath.isNotBlank()) "$relativePath/$filename" else filename 77 | val destDir = "$outputPath/$path" 78 | 79 | // Value is nested map of directories to files and files to the revision for each file. 80 | when (value) { 81 | is Map<*, *> -> { 82 | @Suppress("UNCHECKED_CAST") 83 | syncFiles( 84 | commits = commits, 85 | fileMap = fileMap[filename] as Map, 86 | remote = remote, 87 | repoPath = repoPath, 88 | outputPath = outputPath, 89 | relativePath = path, 90 | verbose = verbose 91 | ) 92 | } 93 | // Value is expected to be the git revision for the file 94 | else -> { 95 | var actualRelativePath = path.substringBeforeLast("/") 96 | if (actualRelativePath == path) { 97 | actualRelativePath = relativePath 98 | } 99 | val revision = if (commits.containsKey(value)) commits[value] else value 100 | // Create the destination directory for each file 101 | // `//file.proto`, 102 | // then run `git show revision:file > dest` to copy the file into the dest 103 | val exitCode = sh( 104 | verbose = verbose, 105 | """ 106 | (cd $repoPath && mkdir -p $outputPath/$actualRelativePath && 107 | (git show $revision:$path > $destDir)) 108 | """.trimIndent() 109 | ) 110 | check(exitCode == 0) { 111 | // Prevent from an empty file in place of the failed synced file. 112 | sh(verbose = verbose, "rm $destDir") 113 | "Failed to sync: $remote/$path: exit code=$exitCode" 114 | } 115 | } 116 | } 117 | } 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /core/src/main/kotlin/com/tinder/gitquery/core/GitQueryInit.kt: -------------------------------------------------------------------------------- 1 | /* 2 | * © 2019 Match Group, LLC. 3 | */ 4 | 5 | package com.tinder.gitquery.core 6 | 7 | import com.tinder.gitquery.core.config.GitQueryConfig 8 | import com.tinder.gitquery.core.utils.insertNested 9 | import com.tinder.gitquery.core.utils.prepareRepo 10 | import com.tinder.gitquery.core.utils.repoHeadRevision 11 | import com.tinder.gitquery.core.utils.sortMap 12 | import java.nio.file.FileSystems 13 | import java.nio.file.Files 14 | import java.nio.file.Path 15 | import java.nio.file.PathMatcher 16 | import java.nio.file.Paths 17 | import java.util.stream.Collectors.toList 18 | 19 | /** 20 | * A utility class to help initialize or update [GitQueryConfig] objects saved in a file. 21 | */ 22 | object GitQueryInit { 23 | private var verbose = false 24 | 25 | /** 26 | * Use the initConfig [GitQueryInitConfig] attribute to initialize or update the [GitQueryConfig] 27 | * and save it to [configFile]. 28 | * 29 | * @param configFile path to the config file. 30 | * @param config a yaml file that describe a set of files to fetch/sync from a given repository. 31 | * @param verbose if true, it will print its operations to standard out. 32 | */ 33 | fun initConfig( 34 | configFile: String, 35 | config: GitQueryConfig, 36 | verbose: Boolean = false, 37 | buildDir: String = System.getProperty("user.dir") + "/build" 38 | ) { 39 | this.verbose = verbose 40 | config.validate() 41 | require(config.initConfig.includeGlobs.isNotEmpty()) { 42 | "Failed to init config: includeGlobs is empty" 43 | } 44 | 45 | val actualRepoDirectory = config.getActualRepoPath(buildDir) 46 | 47 | val initConfig = config.initConfig 48 | 49 | prepareRepo( 50 | config.remote, 51 | if (initConfig.revision.isEmpty()) config.branch else initConfig.revision, 52 | actualRepoDirectory, 53 | verbose = verbose 54 | ) 55 | 56 | config.files = initFiles( 57 | includeGlobs = initConfig.includeGlobs, 58 | excludeGlobs = initConfig.excludeGlobs, 59 | actualRepoDirectory = actualRepoDirectory, 60 | flatFiles = initConfig.flatFiles, 61 | revision = initConfig.revision 62 | ).sortMap() 63 | 64 | config.save(configFile) 65 | println("GitQuery: init complete: $configFile") 66 | } 67 | 68 | @Suppress("ComplexMethod") 69 | private fun initFiles( 70 | includeGlobs: List, 71 | excludeGlobs: List, 72 | actualRepoDirectory: String, 73 | flatFiles: Boolean, 74 | revision: String 75 | ): HashMap { 76 | val actualRepoPath = Paths.get(actualRepoDirectory) 77 | var repoHeadRevision = revision.ifEmpty { repoHeadRevision(actualRepoDirectory, verbose = verbose) } 78 | if (repoHeadRevision.uppercase() == "HEAD") { 79 | repoHeadRevision = repoHeadRevision(actualRepoDirectory, verbose = verbose) 80 | } 81 | 82 | val ret = HashMap() 83 | 84 | val excludeMatchers = ArrayList(excludeGlobs.size) 85 | for (excludeGlob in excludeGlobs) { 86 | excludeMatchers.add(FileSystems.getDefault().getPathMatcher("glob:$excludeGlob")) 87 | } 88 | 89 | for (glob in includeGlobs) { 90 | val matcher = FileSystems.getDefault().getPathMatcher("glob:$glob") 91 | Files.walk(actualRepoPath) 92 | .collect(toList()) 93 | .filter { it: Path? -> 94 | it?.let { path -> 95 | val relativePath = Paths.get(path.toString().substringAfter("$actualRepoDirectory/")) 96 | val matchesInclude = matcher.matches(path) || matcher.matches(relativePath) 97 | var excluded = false 98 | if (matchesInclude) { 99 | for (excludeMatcher in excludeMatchers) { 100 | if (excluded) break 101 | excluded = excludeMatcher.matches(path) || excludeMatcher.matches(relativePath) 102 | } 103 | } 104 | matchesInclude && !excluded 105 | } ?: false 106 | } 107 | .map { 108 | if (verbose) { 109 | println(it) 110 | } 111 | val relativePath = it.toString() 112 | .substringAfter(actualRepoPath.toString()) 113 | .substringAfter("/") 114 | if (flatFiles) { 115 | ret[relativePath] = repoHeadRevision 116 | } else { 117 | ret.insertNested(relativePath, repoHeadRevision) 118 | } 119 | } 120 | } 121 | return ret 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /core/src/test/kotlin/com/tinder/gitquery/core/GitQueryInitTest.kt: -------------------------------------------------------------------------------- 1 | package com.tinder.gitquery.core 2 | 3 | import com.tinder.gitquery.core.config.GitQueryConfig 4 | import com.tinder.gitquery.core.config.GitQueryConfigSchema 5 | import com.tinder.gitquery.core.config.GitQueryInitConfig 6 | import org.junit.After 7 | import org.junit.Before 8 | import org.junit.Rule 9 | import org.junit.Test 10 | import org.junit.rules.TemporaryFolder 11 | import java.io.File 12 | 13 | class GitQueryInitTest { 14 | @get:Rule 15 | val testProjectDir = TemporaryFolder() 16 | 17 | @Before 18 | fun setup() { 19 | testProjectDir.create() 20 | testProjectDir.apply { 21 | newFolder("gitquery-output") 22 | } 23 | } 24 | 25 | @After 26 | fun tearDown() { 27 | testProjectDir.delete() 28 | } 29 | 30 | @Test 31 | fun `initConfig - given missing remote, should fail`() { 32 | // When 33 | val actualError = kotlin.runCatching { 34 | GitQueryInit.initConfig( 35 | configFile = "${testProjectDir.root}/gitquery.yml", 36 | config = GitQueryConfig( 37 | schema = GitQueryConfigSchema(version = "1"), 38 | branch = "master", 39 | files = mapOf( 40 | "definitions" to mapOf("user.proto" to "42933446d0321958e8c12216d04b9f0c382ebf1b") 41 | ), 42 | repoDir = "${testProjectDir.root}/tmp/remote", 43 | outputDir = "${testProjectDir.root}/gitquery-output" 44 | ), 45 | buildDir = testProjectDir.root.path 46 | ) 47 | }.exceptionOrNull() 48 | 49 | // Then 50 | require(actualError is IllegalArgumentException) 51 | assert(actualError.message == "Parameter remote may not be a blank string ()") 52 | 53 | assert(!File("${testProjectDir.root}/gitquery-output/definitions/user.proto").exists()) 54 | assert(!File("${testProjectDir.root}/gitquery-output/README.md").exists()) 55 | } 56 | 57 | @Test 58 | fun `initConfig - given missing branch, should fail`() { 59 | // When 60 | val actualError = kotlin.runCatching { 61 | GitQueryInit.initConfig( 62 | configFile = "${testProjectDir.root}/gitquery.yml", 63 | config = GitQueryConfig( 64 | schema = GitQueryConfigSchema(version = "1"), 65 | remote = "https://github.com/aminghadersohi/ProtoExample.git", 66 | branch = "", 67 | repoDir = "${testProjectDir.root}/tmp/remote", 68 | outputDir = "${testProjectDir.root}/gitquery-output" 69 | ), 70 | buildDir = testProjectDir.root.path 71 | ) 72 | }.exceptionOrNull() 73 | 74 | // Then 75 | require(actualError is IllegalArgumentException) 76 | assert(actualError.message == "Parameter branch may not be a blank string ()") 77 | 78 | assert(!File("${testProjectDir.root}/gitquery-output/definitions/user.proto").exists()) 79 | assert(!File("${testProjectDir.root}/gitquery-output/README.md").exists()) 80 | } 81 | 82 | @Test 83 | fun `initConfig - given empty includeGlobs, should fail`() { 84 | // When 85 | val actualError = kotlin.runCatching { 86 | GitQueryInit.initConfig( 87 | configFile = "${testProjectDir.root}/gitquery.yml", 88 | config = GitQueryConfig( 89 | schema = GitQueryConfigSchema(version = "1"), 90 | remote = "https://github.com/aminghadersohi/ProtoExample.git", 91 | branch = "master", 92 | repoDir = "${testProjectDir.root}/tmp/remote", 93 | outputDir = "${testProjectDir.root}/gitquery-output" 94 | ), 95 | buildDir = testProjectDir.root.path 96 | ) 97 | }.exceptionOrNull() 98 | 99 | // Then 100 | require(actualError is IllegalArgumentException) 101 | assert(actualError.message == "Failed to init config: includeGlobs is empty") 102 | 103 | assert(!File("${testProjectDir.root}/gitquery-output/definitions/user.proto").exists()) 104 | assert(!File("${testProjectDir.root}/gitquery-output/README.md").exists()) 105 | } 106 | 107 | @Test 108 | fun `initConfig - given non-existent file, should create file`() { 109 | // When 110 | GitQueryInit.initConfig( 111 | configFile = "${testProjectDir.root}/gitquery.yml", 112 | config = GitQueryConfig( 113 | schema = GitQueryConfigSchema(version = "1"), 114 | remote = "https://github.com/aminghadersohi/ProtoExample.git", 115 | branch = "master", 116 | repoDir = "${testProjectDir.root}/tmp/remote", 117 | outputDir = "${testProjectDir.root}/gitquery-output", 118 | initConfig = GitQueryInitConfig( 119 | includeGlobs = listOf("**/*.proto") 120 | ) 121 | ), 122 | buildDir = testProjectDir.root.path 123 | ) 124 | 125 | assert(File("${testProjectDir.root}/gitquery.yml").exists()) 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /core/src/main/kotlin/com/tinder/gitquery/core/config/GitQueryConfig.kt: -------------------------------------------------------------------------------- 1 | /* 2 | * © 2019 Match Group, LLC. 3 | */ 4 | 5 | package com.tinder.gitquery.core.config 6 | 7 | import com.tinder.gitquery.core.utils.toAbsolutePath 8 | import org.yaml.snakeyaml.DumperOptions 9 | import org.yaml.snakeyaml.Yaml 10 | import org.yaml.snakeyaml.constructor.Constructor 11 | import java.io.FileWriter 12 | import java.nio.file.Files 13 | import java.nio.file.Paths 14 | import java.util.Collections.emptyMap 15 | 16 | const val DEFAULT_AUTO_SYNC: Boolean = false 17 | const val DEFAULT_BRANCH: String = "" 18 | const val DEFAULT_CLEAN_OUTPUT: Boolean = true 19 | const val DEFAULT_CONFIG_FILENAME: String = "gitquery.yml" 20 | const val DEFAULT_GRADLE_REPO_DIR: String = "tmp/gitquery/repo" 21 | const val DEFAULT_FLAT_FILES: Boolean = false 22 | const val DEFAULT_OUTPUT_DIR: String = "gitquery-output" 23 | const val DEFAULT_REMOTE: String = "" 24 | const val DEFAULT_REPO_DIR: String = "/tmp/gitquery/repo" 25 | const val DEFAULT_REVISION: String = "" 26 | const val DEFAULT_VERBOSE: Boolean = false 27 | val defaultExcludeGlobs: List = emptyList() 28 | val defaultIncludeGlobs: List = emptyList() 29 | 30 | /** 31 | * A model for the config yaml file. 32 | */ 33 | data class GitQueryConfig( 34 | /** The version of the schema for this config. */ 35 | var schema: GitQueryConfigSchema = GitQueryConfigSchema(), 36 | 37 | /** Encapsulate attributes related to init-config. */ 38 | var initConfig: GitQueryInitConfig = GitQueryInitConfig(), 39 | 40 | /** The remote repository to query files from. */ 41 | var remote: String = DEFAULT_REMOTE, 42 | 43 | /** 44 | * The single branch that will be cloned on first run and pulled incrementally on subsequent 45 | * runs. The revisions used in [commits] and [files] must be available under [branch]. 46 | */ 47 | var branch: String = DEFAULT_BRANCH, 48 | 49 | /** 50 | * Specify a nested map of directories to files and file to revisions (or commit alias) included file 51 | * that we want to query and sync. The structure of [files] matches the directory structure of the 52 | * remote repo. A key whose value is a nested map is considered a directory. 53 | */ 54 | var files: Map = emptyMap(), 55 | 56 | /** A list of commit aliases that can be used in the [files] section. */ 57 | var commits: Map = emptyMap(), 58 | 59 | /** A directory to hold the intermediate cloned git repo. */ 60 | var repoDir: String = DEFAULT_REPO_DIR, 61 | 62 | /** A directory to sync the queried files into. */ 63 | var outputDir: String = DEFAULT_OUTPUT_DIR, 64 | 65 | /** If true [default], cleans out the output folder prior to running sync. */ 66 | var cleanOutput: Boolean = DEFAULT_CLEAN_OUTPUT, 67 | 68 | /** A map of String -> Any - enabling self contained integration with various systems. */ 69 | var extra: Map = emptyMap(), 70 | ) { 71 | /** 72 | * Save the config to a file. 73 | * 74 | * @param filename to path to save to. 75 | */ 76 | fun save(filename: String) { 77 | val writer = FileWriter(filename) 78 | val options = DumperOptions() 79 | options.defaultFlowStyle = DumperOptions.FlowStyle.BLOCK 80 | val yaml = Yaml(options) 81 | yaml.dump(this, writer) 82 | writer.flush() 83 | } 84 | 85 | /** 86 | * using the repoDir and remote config attributes, return where the repo should be cloned. 87 | */ 88 | fun getActualRepoPath(pathPrefix: String): String { 89 | val repoPath = toAbsolutePath(path = this.repoDir, prefixPath = pathPrefix) 90 | val repoFullname = remote.substringAfter("git@github.com:") 91 | .substringBefore(".git") 92 | return "$repoPath/$repoFullname" 93 | } 94 | 95 | /** 96 | * Validate essential config attributes. 97 | */ 98 | fun validate() { 99 | require(remote.isNotBlank()) { 100 | "Parameter remote may not be a blank string ($remote)" 101 | } 102 | 103 | require(branch.isNotBlank()) { 104 | "Parameter branch may not be a blank string ($branch)" 105 | } 106 | } 107 | 108 | companion object { 109 | /** 110 | * Load the config file. 111 | * 112 | * @param filename the path to the config file 113 | * @param createIfNotExists if the filename doesn't exist, create it with default attribute values. 114 | */ 115 | fun load(filename: String, createIfNotExists: Boolean): GitQueryConfig { 116 | require(filename.isNotBlank()) { 117 | "Input filename may not be a blank string ($filename)" 118 | } 119 | 120 | val filePath = Paths.get(filename) 121 | if (!Files.exists(filePath) && createIfNotExists) { 122 | GitQueryConfig().save(filename) 123 | } 124 | check(Files.exists(Paths.get(filename))) { 125 | "Config file does not exist ($filename)" 126 | } 127 | 128 | val yaml = Yaml(Constructor(GitQueryConfig::class.java)) 129 | return Files 130 | .newBufferedReader(filePath) 131 | .use { 132 | yaml.load(it) 133 | } 134 | } 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /core/src/test/kotlin/com/tinder/gitquery/core/GitQuerySyncTest.kt: -------------------------------------------------------------------------------- 1 | package com.tinder.gitquery.core 2 | 3 | import com.tinder.gitquery.core.config.GitQueryConfig 4 | import com.tinder.gitquery.core.config.GitQueryConfigSchema 5 | import org.junit.After 6 | import org.junit.Before 7 | import org.junit.Rule 8 | import org.junit.Test 9 | import org.junit.rules.TemporaryFolder 10 | import java.io.File 11 | import java.lang.IllegalStateException 12 | 13 | class GitQuerySyncTest { 14 | @get:Rule 15 | val testProjectDir = TemporaryFolder() 16 | 17 | @Before 18 | fun setup() { 19 | testProjectDir.create() 20 | testProjectDir.apply { 21 | newFolder("gitquery-output") 22 | } 23 | } 24 | 25 | @After 26 | fun tearDown() { 27 | testProjectDir.delete() 28 | } 29 | 30 | @Test 31 | fun `sync - given valid parameters, should succeed`() { 32 | // When 33 | GitQuerySync.sync( 34 | config = GitQueryConfig( 35 | schema = GitQueryConfigSchema(version = "1"), 36 | remote = "https://github.com/aminghadersohi/ProtoExample.git", 37 | branch = "master", 38 | commits = mapOf("latest" to "d654b510d2689e8ee56d23d03dff2be742737f86"), 39 | files = mapOf( 40 | "README.md" to "latest", 41 | "definitions" to mapOf("user.proto" to "42933446d0321958e8c12216d04b9f0c382ebf1b") 42 | ), 43 | repoDir = "${testProjectDir.root}/tmp/remote", 44 | outputDir = "${testProjectDir.root}/gitquery-output" 45 | ), 46 | projectDir = testProjectDir.root.path, 47 | buildDir = testProjectDir.root.path 48 | ) 49 | 50 | // Then 51 | assert(File("${testProjectDir.root}/gitquery-output/definitions/user.proto").exists()) 52 | assert(File("${testProjectDir.root}/gitquery-output/README.md").exists()) 53 | } 54 | 55 | @Test 56 | fun `sync - given missing remote, should fail`() { 57 | // When 58 | val actualError = kotlin.runCatching { 59 | GitQuerySync.sync( 60 | config = GitQueryConfig( 61 | schema = GitQueryConfigSchema(version = "1"), 62 | branch = "master", 63 | files = mapOf( 64 | "definitions" to mapOf("user.proto" to "42933446d0321958e8c12216d04b9f0c382ebf1b") 65 | ), 66 | repoDir = "${testProjectDir.root}/tmp/remote", 67 | outputDir = "${testProjectDir.root}/gitquery-output" 68 | ), 69 | projectDir = testProjectDir.root.path, 70 | buildDir = testProjectDir.root.path 71 | ) 72 | }.exceptionOrNull() 73 | 74 | // Then 75 | require(actualError is IllegalArgumentException) 76 | assert(actualError.message == "Parameter remote may not be a blank string ()") 77 | 78 | assert(!File("${testProjectDir.root}/gitquery-output/definitions/user.proto").exists()) 79 | assert(!File("${testProjectDir.root}/gitquery-output/README.md").exists()) 80 | } 81 | 82 | @Test 83 | fun `sync - given missing branch, should fail`() { 84 | // When 85 | val actualError = kotlin.runCatching { 86 | GitQuerySync.sync( 87 | config = GitQueryConfig( 88 | schema = GitQueryConfigSchema(version = "1"), 89 | remote = "https://github.com/aminghadersohi/ProtoExample.git", 90 | branch = "", 91 | files = mapOf( 92 | "definitions" to mapOf("user.proto" to "42933446d0321958e8c12216d04b9f0c382ebf1b") 93 | ), 94 | repoDir = "${testProjectDir.root}/tmp/remote", 95 | outputDir = "${testProjectDir.root}/gitquery-output" 96 | ), 97 | projectDir = testProjectDir.root.path, 98 | buildDir = testProjectDir.root.path 99 | ) 100 | }.exceptionOrNull() 101 | 102 | // Then 103 | require(actualError is IllegalArgumentException) 104 | assert(actualError.message == "Parameter branch may not be a blank string ()") 105 | 106 | assert(!File("${testProjectDir.root}/gitquery-output/definitions/user.proto").exists()) 107 | assert(!File("${testProjectDir.root}/gitquery-output/README.md").exists()) 108 | } 109 | 110 | @Test 111 | fun `sync - given non-existent file, should fail`() { 112 | // When 113 | val actualError = kotlin.runCatching { 114 | GitQuerySync.sync( 115 | config = GitQueryConfig( 116 | schema = GitQueryConfigSchema(version = "1"), 117 | remote = "https://github.com/aminghadersohi/ProtoExample.git", 118 | branch = "master", 119 | files = mapOf( 120 | "definitions" to mapOf("notexist.proto" to "42933446d0321958e8c12216d04b9f0c382ebf1b") 121 | ), 122 | repoDir = "${testProjectDir.root}/tmp/remote", 123 | outputDir = "${testProjectDir.root}/gitquery-output" 124 | ), 125 | projectDir = testProjectDir.root.path, 126 | buildDir = testProjectDir.root.path 127 | ) 128 | }.exceptionOrNull() 129 | 130 | // Then 131 | require(actualError is IllegalStateException) 132 | assert( 133 | actualError.message == "Failed to sync: " + 134 | "https://github.com/aminghadersohi/ProtoExample.git/definitions/notexist.proto: exit code=128" 135 | ) 136 | 137 | assert(!File("${testProjectDir.root}/gitquery-output/definitions/notexist.proto").exists()) 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | ############################################################################## 4 | ## 5 | ## Gradle start up script for UN*X 6 | ## 7 | ############################################################################## 8 | 9 | # Attempt to set APP_HOME 10 | # Resolve links: $0 may be a link 11 | PRG="$0" 12 | # Need this for relative symlinks. 13 | while [ -h "$PRG" ] ; do 14 | ls=`ls -ld "$PRG"` 15 | link=`expr "$ls" : '.*-> \(.*\)$'` 16 | if expr "$link" : '/.*' > /dev/null; then 17 | PRG="$link" 18 | else 19 | PRG=`dirname "$PRG"`"/$link" 20 | fi 21 | done 22 | SAVED="`pwd`" 23 | cd "`dirname \"$PRG\"`/" >/dev/null 24 | APP_HOME="`pwd -P`" 25 | cd "$SAVED" >/dev/null 26 | 27 | APP_NAME="Gradle" 28 | APP_BASE_NAME=`basename "$0"` 29 | 30 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 31 | DEFAULT_JVM_OPTS='"-Xmx64m"' 32 | 33 | # Use the maximum available, or set MAX_FD != -1 to use that value. 34 | MAX_FD="maximum" 35 | 36 | warn () { 37 | echo "$*" 38 | } 39 | 40 | die () { 41 | echo 42 | echo "$*" 43 | echo 44 | exit 1 45 | } 46 | 47 | # OS specific support (must be 'true' or 'false'). 48 | cygwin=false 49 | msys=false 50 | darwin=false 51 | nonstop=false 52 | case "`uname`" in 53 | CYGWIN* ) 54 | cygwin=true 55 | ;; 56 | Darwin* ) 57 | darwin=true 58 | ;; 59 | MINGW* ) 60 | msys=true 61 | ;; 62 | NONSTOP* ) 63 | nonstop=true 64 | ;; 65 | esac 66 | 67 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 68 | 69 | # Determine the Java command to use to start the JVM. 70 | if [ -n "$JAVA_HOME" ] ; then 71 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 72 | # IBM's JDK on AIX uses strange locations for the executables 73 | JAVACMD="$JAVA_HOME/jre/sh/java" 74 | else 75 | JAVACMD="$JAVA_HOME/bin/java" 76 | fi 77 | if [ ! -x "$JAVACMD" ] ; then 78 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 79 | 80 | Please set the JAVA_HOME variable in your environment to match the 81 | location of your Java installation." 82 | fi 83 | else 84 | JAVACMD="java" 85 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 86 | 87 | Please set the JAVA_HOME variable in your environment to match the 88 | location of your Java installation." 89 | fi 90 | 91 | # Increase the maximum file descriptors if we can. 92 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then 93 | MAX_FD_LIMIT=`ulimit -H -n` 94 | if [ $? -eq 0 ] ; then 95 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 96 | MAX_FD="$MAX_FD_LIMIT" 97 | fi 98 | ulimit -n $MAX_FD 99 | if [ $? -ne 0 ] ; then 100 | warn "Could not set maximum file descriptor limit: $MAX_FD" 101 | fi 102 | else 103 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 104 | fi 105 | fi 106 | 107 | # For Darwin, add options to specify how the application appears in the dock 108 | if $darwin; then 109 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 110 | fi 111 | 112 | # For Cygwin, switch paths to Windows format before running java 113 | if $cygwin ; then 114 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 115 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 116 | JAVACMD=`cygpath --unix "$JAVACMD"` 117 | 118 | # We build the pattern for arguments to be converted via cygpath 119 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 120 | SEP="" 121 | for dir in $ROOTDIRSRAW ; do 122 | ROOTDIRS="$ROOTDIRS$SEP$dir" 123 | SEP="|" 124 | done 125 | OURCYGPATTERN="(^($ROOTDIRS))" 126 | # Add a user-defined pattern to the cygpath arguments 127 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 128 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 129 | fi 130 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 131 | i=0 132 | for arg in "$@" ; do 133 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 134 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 135 | 136 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 137 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 138 | else 139 | eval `echo args$i`="\"$arg\"" 140 | fi 141 | i=$((i+1)) 142 | done 143 | case $i in 144 | (0) set -- ;; 145 | (1) set -- "$args0" ;; 146 | (2) set -- "$args0" "$args1" ;; 147 | (3) set -- "$args0" "$args1" "$args2" ;; 148 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;; 149 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 150 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 151 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 152 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 153 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 154 | esac 155 | fi 156 | 157 | # Escape application args 158 | save () { 159 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done 160 | echo " " 161 | } 162 | APP_ARGS=$(save "$@") 163 | 164 | # Collect all arguments for the java command, following the shell quoting and substitution rules 165 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" 166 | 167 | # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong 168 | if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then 169 | cd "$(dirname "$0")" 170 | fi 171 | 172 | exec "$JAVACMD" "$@" 173 | -------------------------------------------------------------------------------- /cli/src/main/kotlin/com/tinder/gitquery/cli/GitQueryCli.kt: -------------------------------------------------------------------------------- 1 | /* 2 | * © 2019 Match Group, LLC. 3 | */ 4 | 5 | package com.tinder.gitquery.cli 6 | 7 | import com.github.ajalt.clikt.core.CliktCommand 8 | import com.github.ajalt.clikt.parameters.options.default 9 | import com.github.ajalt.clikt.parameters.options.flag 10 | import com.github.ajalt.clikt.parameters.options.option 11 | import com.github.ajalt.clikt.parameters.options.versionOption 12 | import com.tinder.gitquery.core.GIT_QUERY_VERSION 13 | import com.tinder.gitquery.core.GitQueryInit 14 | import com.tinder.gitquery.core.GitQuerySync 15 | import com.tinder.gitquery.core.config.DEFAULT_BRANCH 16 | import com.tinder.gitquery.core.config.DEFAULT_CLEAN_OUTPUT 17 | import com.tinder.gitquery.core.config.DEFAULT_CONFIG_FILENAME 18 | import com.tinder.gitquery.core.config.DEFAULT_FLAT_FILES 19 | import com.tinder.gitquery.core.config.DEFAULT_OUTPUT_DIR 20 | import com.tinder.gitquery.core.config.DEFAULT_REMOTE 21 | import com.tinder.gitquery.core.config.DEFAULT_REPO_DIR 22 | import com.tinder.gitquery.core.config.DEFAULT_VERBOSE 23 | import com.tinder.gitquery.core.config.GitQueryConfig 24 | 25 | class GitQueryCli : CliktCommand() { 26 | init { 27 | versionOption(GIT_QUERY_VERSION, message = { version -> "GitQuery $version" }) 28 | } 29 | 30 | private val configFile: String by option( 31 | help = """A yaml file that describe a set of files to query and sync from a given repository. 32 | |default: $DEFAULT_CONFIG_FILENAME""".trimMargin() 33 | ).default(DEFAULT_CONFIG_FILENAME) 34 | 35 | // Override attributes. The following attributes override the same value in config is they are defined. 36 | private val remote: String by option( 37 | help = """Remote Git repo url. 38 | |If provided, this will override any value specified for [remote] in the [configFile]. 39 | |default: $DEFAULT_REMOTE""".trimMargin() 40 | ).default("") 41 | 42 | private val branch: String by option( 43 | help = """Remote Git repo branch. 44 | |If provided, this will override any value specified for [branch] in the [configFile]. 45 | |default: $DEFAULT_BRANCH""".trimMargin() 46 | ).default("") 47 | 48 | private val outputDir: String by option( 49 | help = """Path to a directory where the files should be synced to. 50 | |If provided, this will override any value specified for [outputDir] in the [configFile]. 51 | |default: $DEFAULT_OUTPUT_DIR""".trimMargin() 52 | ).default("") 53 | 54 | private val repoDir: String by option( 55 | help = """Where the remote repo(s) can be cloned locally and stored. 56 | |If provided, this will override any value specified for [repoDir] in the [configFile]. 57 | |default: $DEFAULT_REPO_DIR""".trimMargin() 58 | ).default("") 59 | 60 | private val dontCleanOutput: Boolean by option( 61 | help = """Whether to not clean the output folder prior to sync. 62 | |If set, this will override the [cleanOutput] in the [configFile] 63 | |default: ${!DEFAULT_CLEAN_OUTPUT}""".trimMargin() 64 | ).flag(default = !DEFAULT_CLEAN_OUTPUT) 65 | 66 | private val initConfig: Boolean by option( 67 | help = """Initialize/update the config file based on command line params. 68 | |Use --include-globs and --exclude-globs. 69 | |If [configFile] exists, it will be updated, else it will be created with values 70 | |from command line or internal defaults. 71 | |default: false""".trimMargin() 72 | ).flag(default = false) 73 | 74 | private val includeGlobs: String by option( 75 | help = """A list of globs to include when generating/updating the files attribute in [configFile]. 76 | |If provided, this comma, space or pipe separated list of globs will override [includeGlobs] 77 | |in [configFile] and used when initializing/updating the config's [files] map.""".trimMargin() 78 | ).default("") 79 | 80 | private val excludeGlobs: String by option( 81 | help = """A list of globs to exclude when generating/updating the files attribute in [configFile]. 82 | |If provided, this comma, space or pipe separated list of globs will override [excludeGlobs] 83 | |in [configFile] and used to exclude patterns when initializing/updating 84 | |the config's [files] map.""".trimMargin() 85 | ).default("") 86 | 87 | private val flatFiles: Boolean by option( 88 | help = """When --generate-globs is used, this option helps choose if the files in 89 | |the generated config file should be in a flat map or a nest map. 90 | |default: $DEFAULT_FLAT_FILES""".trimMargin() 91 | ).flag(default = DEFAULT_FLAT_FILES) 92 | 93 | private val revision: String by option( 94 | help = """A revision to use when --init-config is used, 95 | |if not provided the revision of latest [branch] is used""".trimMargin() 96 | ).default("") 97 | 98 | private val verbose: Boolean by option( 99 | help = """Show the underlying commands and their outputs in the console. 100 | |default: $DEFAULT_VERBOSE""".trimMargin() 101 | ).flag(default = DEFAULT_VERBOSE) 102 | 103 | override fun run() { 104 | val config = GitQueryConfig.load(configFile, initConfig) 105 | if (remote.isNotBlank()) { 106 | config.remote = remote 107 | } 108 | if (branch.isNotBlank()) { 109 | config.branch = branch 110 | } 111 | if (repoDir.isNotBlank()) { 112 | config.repoDir = repoDir 113 | } 114 | if (outputDir.isNotBlank()) { 115 | config.outputDir = outputDir 116 | } 117 | config.cleanOutput = !dontCleanOutput 118 | if (initConfig) { 119 | if (includeGlobs.isNotBlank()) { 120 | config.initConfig.includeGlobs = includeGlobs.split(",", " ", "|") 121 | } 122 | if (excludeGlobs.isNotBlank()) { 123 | config.initConfig.excludeGlobs = excludeGlobs.split(",", " ", "|") 124 | } 125 | if (revision.isNotBlank()) { 126 | config.initConfig.revision = revision 127 | } 128 | config.initConfig.flatFiles = flatFiles 129 | 130 | GitQueryInit.initConfig(configFile = configFile, config = config, verbose = verbose) 131 | } else { 132 | GitQuerySync.sync(config = config, verbose = verbose) 133 | } 134 | } 135 | } 136 | 137 | fun main(args: Array) = GitQueryCli().main(args) 138 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | GitQuery 2 | ========== 3 | 4 | Fast and incremental query and sync of files in a remote git repository. 5 | 6 | - Core library (Kotlin) 7 | - Command line interface 8 | - Gradle plugin 9 | 10 | An alternative approach for defining dependencies in a build project vs importing of pre-built and published artifacts. 11 | 12 | #### Use cases: 13 | It is sometimes preferable to query individual files in a remote repo when: 14 | - we only need a few files from large remote monorepo. 15 | - we need granular versioning for each dependency file. 16 | - it is hard to globally version the files in the remote repo, or it would require many artifacts to be published. 17 | - we want to combine the source files of an artifact into our module. 18 | - we want to avoid version conflicts as possible when including pre-build artifacts. 19 | 20 | #### Sample gitquery.yml 21 | 22 | ```yaml 23 | # Describe a set of files to fetch from a given repository. 24 | --- 25 | schema: 26 | version: 1 27 | # remote - The remote repository to query files from. 28 | remote: https://github.com/aminghadersohi/ProtoExample.git 29 | # branch - The single branch that will be cloned on first run and pulled incrementally on 30 | # subsequent runs. The revision values used in [commits] and [files] must be available under [branch]. 31 | branch: master 32 | # A list of commit aliases that can be used in the [files] section. 33 | commits: 34 | latest: d654b510d2689e8ee56d23d03dff2be742737f86 35 | # Specify a nested map of filenames to revisions (or commit alias) that we want to sync to [outputDir]. 36 | # The structure of [files] matches the directory structure of the remote repo. A key whose value is a 37 | # nested map is considered a directory. 38 | files: 39 | # A file at the root fo the remote repo. 40 | README.md: latest 41 | # A directory at the root of the remote repo. 42 | definitions: 43 | # A file inside the definitions folder. 44 | user.proto: 42933446d0321958e8c12216d04b9f0c382ebf1b 45 | # A directory to sync the queried files into. default: gitquery-output 46 | outputDir: gitquery-output 47 | # A directory to hold the intermediate cloned git repo. default: /tmp/gitquery/repo 48 | repoDir: /tmp/.gitquery 49 | # If true [default], cleans out the output folder prior to running sync. 50 | cleanOutput: true 51 | # A free form map of String -> Any - enabling self contained integration with various systems. 52 | extra: 53 | attr1: value1 54 | attr2: 1 55 | # 56 | ``` 57 | 58 | #### Gradle Plugin - (https://plugins.gradle.org/plugin/com.tinder.gitquery) 59 | `module/build.gradle:` 60 | ```groovy 61 | plugins { 62 | id "com.tinder.gitquery" version "3.0.12" 63 | } 64 | 65 | gitQuery { 66 | autoSync = true 67 | cleanOutput = true 68 | configFile = "gitquery.yml" 69 | outputDir = protoDir 70 | repoDir = "${project.buildDir}/tmp/.gitquery" 71 | } 72 | 73 | // This will init/update the config file every time the gitQuery task runs. 74 | gitQueryInit { 75 | flatFiles = false 76 | includeGlobs = listOf("**/examples/addressbook.proto") 77 | excludeGlobs = listOf("**/generated/**/*.proto") 78 | revision = "v3.14.0" 79 | } 80 | ``` 81 | 82 | This adds a task called `gitQuery` to the module. It can be executed like so: 83 | 84 | ```terminal 85 | ./gradlew :module:gitQuery 86 | ``` 87 | 88 | #### Bazel 89 | 90 | Add the needed dependencies to your `WORKSPACE` file: 91 | 92 | ```WORKSPACE 93 | load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") 94 | 95 | http_archive( 96 | name = "GitQuery", 97 | url = "RELEASE_URL", 98 | sha256 = "UPDATE_ME" 99 | ) 100 | 101 | load("@GitQuery//bazel_support:repositories.bzl", "gitquery_dependencies") 102 | load("@GitQuery//bazel_support:constants.bzl", "MAVEN_ARTIFACTS") 103 | 104 | gitquery_dependencies() 105 | 106 | load("@io_bazel_rules_kotlin//kotlin:repositories.bzl", "kotlin_repositories") 107 | kotlin_repositories() 108 | 109 | load("@io_bazel_rules_kotlin//kotlin:core.bzl", "kt_register_toolchains") 110 | kt_register_toolchains() 111 | 112 | load("@rules_jvm_external//:defs.bzl", "maven_install") 113 | 114 | maven_install( 115 | artifacts = MAVEN_ARTIFACTS, 116 | repositories = [ 117 | "https://maven.google.com", 118 | "https://repo1.maven.org/maven2", 119 | ], 120 | ) 121 | ``` 122 | 123 | Then run 124 | 125 | ```terminal 126 | bazel run @GitQuery//:GitQuery -- -h 127 | ``` 128 | 129 | to get started. 130 | 131 | #### Command line interface 132 | Install using brew (https://github.com/Tinder/homebrew-tap): 133 | ``` 134 | brew tap Tinder/tap 135 | brew install gitquery 136 | ``` 137 | or 138 | ``` 139 | brew install Tinder/tap/gitquery 140 | ``` 141 | 142 | If you need Java 11: 143 | ``` 144 | brew install --cask homebrew/cask-versions/temurin11 145 | ``` 146 | 147 | To install from source: 148 | ```shell script 149 | ./install 150 | ``` 151 | 152 | To run 153 | ```shell script 154 | gitquery 155 | ``` 156 | 157 | #### Sample Sync 158 | ```shell script 159 | ./gitquery --config-file=./samples/sample1.yml --repo-dir=./build/tmp/repo --output-dir=./gitquery-output 160 | ``` 161 | 162 | #### Sample Init - create config that doesn't exist, by default a nested structure generated. 163 | ```shell script 164 | ./gitquery --init-config \ 165 | --config-file samples/sample1-generate-nested.yml \ 166 | --include-globs "**/src/google/protobuf/**/*.proto,**/benchmarks/**/*.proto,README.md,benchmarks/Makefile.am" \ 167 | --exclude-globs "**/ruby/**,**/proto2/**" \ 168 | --remote git@github.com:protocolbuffers/protobuf.git \ 169 | --branch master \ 170 | --revision v3.14.0 171 | ``` 172 | 173 | #### Sample Update - by default a nested structure generated. 174 | ```shell script 175 | ./gitquery --init-config \ 176 | --config-file samples/sample2-generate-nested.yml \ 177 | --include-globs "**/src/google/protobuf/**/*.proto,**/benchmarks/**/*.proto,README.md" \ 178 | --exclude-globs "**/ruby/**,**/proto2/**" \ 179 | --revision v3.14.0 180 | ``` 181 | 182 | #### Sample Update - flat file structure generated using --flat-files. 183 | ```shell script 184 | ./gitquery --init-config \ 185 | --config-file samples/sample2-generate-flat.yml --flat-files \ 186 | --include-globs "**/src/google/protobuf/**/*.proto,**/benchmarks/**/*.proto,README.md" \ 187 | --exclude-globs "**/ruby/**,**/proto2/**"\ 188 | --revision v3.14.0 189 | ``` 190 | 191 | ```shell script 192 | ./gitquery --help 193 | 194 | Usage: git-query-cli [OPTIONS] 195 | 196 | Options: 197 | --version Show the version and exit 198 | --config-file TEXT A yaml file that describe a set of files to query and 199 | sync from a given repository. default: gitquery.yml 200 | --remote TEXT Remote Git repo url. If provided, this will override 201 | any value specified for [remote] in the [configFile]. 202 | default: 203 | --branch TEXT Remote Git repo branch. If provided, this will 204 | override any value specified for [branch] in the 205 | [configFile]. default: master 206 | --output-dir TEXT Path to a directory where the files should be synced 207 | to. If provided, this will override any value 208 | specified for [outputDir] in the [configFile]. 209 | default: gitquery-output 210 | --repo-dir TEXT Where the remote repo(s) can be cloned locally and 211 | stored. If provided, this will override any value 212 | specified for [repoDir] in the [configFile]. default: 213 | /tmp/gitquery/repo 214 | --dont-clean-output Whether to not clean the output folder prior to sync. 215 | If set, this will override the [cleanOutput] in the 216 | [configFile] default: false 217 | --init-config Initialize/update the config file based on command 218 | line params. Use --include-globs and --exclude-globs. 219 | If [configFile] exists, it will be updated, else it 220 | will be created with values from command line or 221 | internal defaults. default: false 222 | --include-globs TEXT A list of globs to include when generating/updating 223 | the files attribute in [configFile]. If provided, this 224 | comma, space or pipe separated list of globs will 225 | override [includeGlobs] in [configFile] and used when 226 | initializing/updating the config's [files] map. 227 | --exclude-globs TEXT A list of globs to exclude when generating/updating 228 | the files attribute in [configFile]. If provided, this 229 | comma, space or pipe separated list of globs will 230 | override [excludeGlobs] in [configFile] and used to 231 | exclude patterns when initializing/updating the 232 | config's [files] map. 233 | --flat-files When --generate-globs is used, this option helps 234 | choose if the files in the generated config file 235 | should be in a flat map or a nest map. default: false 236 | --revision TEXT A revision to use when --init-config is used, if not 237 | provided the revision of latest [branch] is used 238 | --verbose Show the underlying commands and their outputs in the 239 | console. default: false 240 | -h, --help Show this message and exit 241 | ``` 242 | 243 | #### Example Use Case - Code Generation 244 | We have a tech stack with many mobile/web apps and services with many developers actively working on them. We want to avoid duplicating and handwriting our models and service interfaces, so we decide to store definitions of models and service interfaces in a central idl repo. 245 | 246 | We can choose to build a single artifact for each language that we support in our tech stack or even build many of them. However, since each of our apps or services just needs a couple of files from the idl repo. We can avoid including unneccesary files, in pre-built artifacts, by using GitQuery to pull and sync the idl files that we want. This allows them to be staged for our build. From there we can use our code generation tools, the sames ones we were using to generate code and build them in to an artifact, to generate code from the idl definitions that we care about and use the generated code in our project as any other source code. 247 | 248 | License 249 | --- 250 | ~~~ 251 | Copyright (c) 2021, Match Group, LLC 252 | All rights reserved. 253 | 254 | Redistribution and use in source and binary forms, with or without 255 | modification, are permitted provided that the following conditions are met: 256 | * Redistributions of source code must retain the above copyright 257 | notice, this list of conditions and the following disclaimer. 258 | * Redistributions in binary form must reproduce the above copyright 259 | notice, this list of conditions and the following disclaimer in the 260 | documentation and/or other materials provided with the distribution. 261 | * Neither the name of Match Group, LLC nor the names of its contributors 262 | may be used to endorse or promote products derived from this software 263 | without specific prior written permission. 264 | 265 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 266 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 267 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 268 | DISCLAIMED. IN NO EVENT SHALL MATCH GROUP, LLC BE LIABLE FOR ANY 269 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 270 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 271 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 272 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 273 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 274 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 275 | --------------------------------------------------------------------------------