├── .gitignore ├── .travis.yml ├── CHANGELOG.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── dev-resources ├── logback-silent.xml └── logback.xml ├── doc └── MIGRATION_GUIDE.md ├── example └── lambdacd_git │ └── example │ ├── multi_repo_pipeline.clj │ └── simple_pipeline.clj ├── go ├── project.clj ├── scripts ├── github-release.sh ├── id_rsa_gitlab-test.enc └── travis_prebuild.sh ├── src └── lambdacd_git │ ├── core.clj │ ├── git.clj │ ├── ssh.clj │ └── ssh_agent_support.clj └── test └── lambdacd_git ├── core_test.clj ├── end_to_end_test.clj ├── git_test.clj ├── git_utils.clj ├── ssh_test.clj └── test_utils.clj /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /classes 3 | /checkouts 4 | pom.xml 5 | pom.xml.asc 6 | *.jar 7 | *.class 8 | /.lein-* 9 | /.nrepl-port 10 | .hgignore 11 | .hg/ 12 | .idea 13 | *.iws 14 | *.iml 15 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | before_script: 2 | - "./scripts/travis_prebuild.sh" 3 | language: clojure 4 | script: "./go test" 5 | cache: 6 | directories: 7 | - "$HOME/.m2" 8 | env: 9 | matrix: 10 | - LAMBDACD_VERSION=0.14.0 CLOJURE_VERSION=1.8.0 11 | - LAMBDACD_VERSION=0.14.0 CLOJURE_VERSION=1.9.0 12 | - LAMBDACD_VERSION=0.13.5 CLOJURE_VERSION=1.7.0 13 | - LAMBDACD_VERSION=0.13.5 CLOJURE_VERSION=1.8.0 14 | - LAMBDACD_VERSION=0.13.4 CLOJURE_VERSION=1.7.0 15 | - LAMBDACD_VERSION=0.13.4 CLOJURE_VERSION=1.8.0 16 | - LAMBDACD_VERSION=0.13.3 CLOJURE_VERSION=1.7.0 17 | - LAMBDACD_VERSION=0.13.3 CLOJURE_VERSION=1.8.0 18 | global: 19 | secure: "L902EMwhFyNeSBRDfIk2BWiQ07caZmjMAhMo12l9K31EGlK9OQRzYs9N+Wz5dWfzkss2PxSpAU/7h9zSvmTZS9FnGnpKTTiki97t9AiRIAllcRt7P+rFsWvdTblY06zQvS/9JOgJ2eAu0IPoD9pLPFGJbLc0DyNkAfsVz3sA/dbow7zIetNvmcHupalcuJidaGze3/wtoxFfW6j3rS+5lr12QLF3KqA2j+Zlk4uDOWfB1wm168MGgBNkpruHhbbMyujNylnK5f50BX0euE/jX+khoDKjFGey3QyB+lIur9mpoe6ZoBGMLxVbVATGRwgDGoY4+JGmo0ItK6e33m9kRLaU9nLDyrojOnff+oKJjVISx4x2k+T1zgAou+d2Tqzz5t1hSSJgs7hyj/N902QOoFH5aJShlgOTtkaAAD4rs9htQrDcCDfcjBbKCFwAurrzEbcyy8CirlASMlSJyjSCOff21qOkoYFCCrwdd5ASz466QWGWmR/7I2D/xfXz303XV1wv642HDqGRnwhqAisstpZKuKNhwmqkwamONbiRMYJB8DFUeWAl4HLA9dda1GBEEwmPRJPohB/64nYjPbASu29L0YF0V5noQ3dcJUf+LdDi1DhndvkEkSLbesFj4ogxk94F3jzSUAzUx+01XU4kTK9+hJssIoAcj2U8Ls8ERUY=" 20 | addons: 21 | ssh_known_hosts: gitlab.com 22 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | This changelog contains a loose collection of changes in every release. I will also try and document all breaking changes to the API. 3 | 4 | The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to a "shifted" version of semantic versioning while the major version remains at 0: Minor version changes indicate breaking changes, patch version changes should not contain breaking changes. 5 | 6 | ## 0.4.1 7 | 8 | ### Added 9 | 10 | * The `:ssh :identity-file` configuration can now load any file as an identity file. 11 | 12 | Prior to this change it would only load `id_rsa`, `id_dsa` and `identity` in `~/.ssh`. 13 | * Support for Clojure 1.9 14 | 15 | 16 | ## 0.4.0 17 | 18 | ### Changed 19 | 20 | * Requires LambdaCD 0.13.5 that no longer depends on insecure repositories to work in newer Leiningen versions (see flosell/lambdacd#171) 21 | 22 | ## 0.3.0 23 | 24 | ### Added 25 | 26 | * Added configuration option for `StrictHostKeyChecking`. Overrides the default in `~/.ssh/config` 27 | 28 | ### Changed 29 | 30 | * Consolidated configuration (e.g. timeouts, ssh options, ...): lambdacd-git can now be configured through LambdaCDs config map and configuration can be overridden per call using function arguments. 31 | 32 | Configuration (e.g. timeouts) that were previously only possible for some functions are now available throughout. SSH config that could previously only be defined for the whole JVM can now be configured per pipeline (through the config map) and even per step (through function parameters). 33 | 34 | Using the config map and `init-ssh!` at the same time will result in runtime errors so make sure you migrate configuration and remove calls to `init-ssh!.` 35 | 36 | See README for details 37 | * Breaking changes in utility namespace `lambdacd-git.git`: Removed keyword arguments and replaced them with an optional options-map in the following functions: 38 | * `lambdacd-git.git/current-revisions` 39 | * `lambdacd-git.git/clone-repo` 40 | * `lambdacd-git.git/push` 41 | 42 | ### Deprecated 43 | 44 | * `lambdacd-git.core/init-ssh!` has been replaced by config via config-map (see above) and will be removed in future releases. 45 | * `lambdacd-git.ssh/session-factory [jsch-customizer-fns]` now also receives `session-customizer-fns`; the single argument function will be removed in future releases 46 | 47 | ### Removed 48 | 49 | * The following deprected functions have been removed: 50 | * `lambdacd-git.ssh-agent-support/session-factory` 51 | * `lambdacd-git.ssh-agent-support/initialize-ssh-agent-support!` 52 | 53 | ## 0.2.1 54 | 55 | ### Added 56 | 57 | * Support authentication with HTTPS using JGits [`CredentialsProvider`](http://download.eclipse.org/jgit/site/4.1.1.201511131810-r/apidocs/org/eclipse/jgit/transport/CredentialsProvider.html). See [README](./README.md) for details 58 | 59 | ### Fixed 60 | 61 | * Fixed NullPointerException in case no `known_hosts` file exists (#21) 62 | 63 | ## 0.2.0 64 | 65 | ### Added 66 | 67 | * Adds compatibility with LambdaCD versions 0.12.0 and greater. Incompatible with versions earlier than 0.9.1. (#19) 68 | 69 | ## 0.1.6 70 | 71 | ### Added 72 | * Add support for git tag and push. 73 | Use case is to tag a deployed version to a commit. 74 | Push pushes all new tags and commits to a given repository. 75 | (Thanks to @rohte) 76 | 77 | ## 0.1.5 78 | 79 | ### Added 80 | * Allow setting of a specific identity file (#12). 81 | Use case is to select the desired account from multiple GitHub accounts that have differing private repo membership. 82 | (Thanks to @markdingram) 83 | 84 | ## 0.1.4 85 | 86 | ### Fixed 87 | * Fixed bug in error handling in `wait-for-git` (#15) 88 | (Thanks to @ImmoStanke) 89 | 90 | ## 0.1.3 91 | 92 | ### Added 93 | * Allow specifying a timeout when cloning a repository 94 | 95 | ## 0.1.2 96 | 97 | ### Added 98 | 99 | * Made step-killing behavior of `wait-for-git` independent of polling frequency (#10) 100 | * Supporting known hosts files other than `~/.ssh/known_hosts` (e.g. `/etc/ssh/ssh_known_hosts`). (#9) 101 | 102 | ### Deprecated 103 | 104 | * `lambdcd-git/ssh-agent-support/initialize-ssh-agent-support!` is deprecated and will be removed in subsequent releases. 105 | It is being replaced with `lambdacd-git.core/init-ssh!` 106 | 107 | ## 0.1.1 108 | 109 | ### Added 110 | 111 | * Added a way to notify `wait-for-git` through HTTP POST requests (#6) 112 | 113 | ## 0.1.0 114 | 115 | * Initial Release 116 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contribution Guide 2 | 3 | ## Contributions encouraged 4 | 5 | I'd love to hear from you! If you have a question, bug report, feature request or a pull request, please reach out. 6 | 7 | ## How to reach out 8 | 9 | The preferred way at the moment is to open issues on the [Github Issue Tracker](https://github.com/flosell/lambdacd/issues) 10 | 11 | If you want to contribute improvements to the codebase, open a pull request. 12 | 13 | ## How to open the perfect issue 14 | 15 | * Be specific and as detailed as you feel is necessary to understand the topic 16 | * Provide context (what were you trying to achive, what were you expecting, ...) 17 | * Code samples and logs can be really helpful. Consider [Gists](https://gist.github.com/) or links to other Github repos 18 | for larger pieces. 19 | * If you are reporting a bug, add steps to reproduce it. 20 | 21 | ## How to create the perfect pull request 22 | 23 | * Have a look into the [`README`](README.md#development) for details on how to work with the 24 | code 25 | * Follow the usual best practices for pull requests: 26 | * use a branch 27 | * make sure you have pulled changes from upstream so that your change is easy to merge 28 | * follow the conventions in the code 29 | * keep a tidy commit history that speaks for itself, consider squashing commits where appropriate 30 | * Run all the tests: `./go test` 31 | * Add tests where possible 32 | * Add an entry in [`CHANGELOG.md`](CHANGELOG.md) if you add new features, fix bugs or otherwise change LambdaCD in a way that you want 33 | users to be aware of. The entry goes into the section for the next release (which is the version number indicated in 34 | `project.clj`), usually the top one. If that section doesn't exist yet, add it. 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Git support for LambdaCD 2 | 3 | Provides Git support for [LambdaCD](https://github.com/flosell/lambdacd). 4 | Will replace the `lambdacd.steps.git` namespace in the the lambdacd-git library. 5 | 6 | ## Status 7 | 8 | [![Build Status](https://travis-ci.org/flosell/lambdacd-git.svg)](https://travis-ci.org/flosell/lambdacd-git) 9 | 10 | [![Clojars Project](http://clojars.org/lambdacd-git/latest-version.svg)](http://clojars.org/lambdacd-git) 11 | 12 | ## Usage 13 | 14 | ```clojure 15 | ;; project.clj 16 | :dependencies [[lambdacd-git ""]] 17 | ;; import: 18 | (:require [lambdacd-git.core :as lambdacd-git]) 19 | ``` 20 | 21 | ### Complete Example 22 | 23 | You'll find a complete example here: [example/simple_pipeline.clj](https://github.com/flosell/lambdacd-git/blob/master/example/lambdacd_git/example/simple_pipeline.clj) 24 | 25 | ### Waiting for a commit 26 | 27 | ```clojure 28 | (defn wait-for-commit-on-master [args ctx] 29 | (lambdacd-git/wait-for-git ctx "git@github.com:flosell/testrepo" 30 | ; how long to wait when polling. optional, defaults to 10000 31 | :ms-between-polls 1000 32 | ; which refs to react to. optional, defaults to refs/heads/master 33 | :ref "refs/heads/master")) 34 | 35 | ; you can also pass in a regex: 36 | (defn wait-for-commit-on-feature-branch [args ctx] 37 | (lambdacd-git/wait-for-git ctx "git@github.com:flosell/testrepo" 38 | :ref #"refs/heads/feature-.*")) 39 | 40 | ; you can also pass in a function 41 | (defn wait-for-commit-on-any-tag [args ctx] 42 | (lambdacd-git/wait-for-git ctx "git@github.com:flosell/testrepo" 43 | :ref (fn [ref] (.startsWith ref "refs/tags/")))) 44 | ``` 45 | 46 | ### Using Web- or Post-Commit Hooks instead of Polling 47 | 48 | `wait-for-git` can be notified about changes in a repository through an HTTP endpoint. To use this, you need to add it 49 | to your existing ring-handlers: 50 | 51 | ```clojure 52 | (ring-server/serve (routes 53 | (ui/ui-for pipeline) 54 | (core/notifications-for pipeline)) ; <-- THIS 55 | {:open-browser? false 56 | :port 8082}) 57 | ``` 58 | 59 | This adds an HTTP endpoint that can receive POST requests on `/notify-git?remote=`, 60 | e.g. `http://localhost:8082/notify-git?remote=git@github.com:flosell/testrepo` 61 | 62 | Notice that this doesn't necessarily trigger a new build. Similar to [how Jenkins works](https://wiki.jenkins.io/display/JENKINS/Git+Plugin#GitPlugin-Pushnotificationfromrepository), a notification just forces a check for changes. 63 | 64 | ### Cloning a Repository 65 | 66 | ```clojure 67 | (defn clone [args ctx] 68 | (lambdacd-git/clone ctx repo branch-or-tag-or-commit-hash (:cwd args))) 69 | 70 | (def pipeline-structure 71 | `(; ... 72 | (with-workspace 73 | clone 74 | do-something))) 75 | 76 | ; Works well with wait-for-git: 77 | ; If no revision is given (e.g. because of manual trigger), clone falls back to the head of the master branch 78 | 79 | (defn clone [args ctx] 80 | (lambdacd-git/clone ctx repo (:revision args) (:cwd args))) 81 | 82 | (def pipeline-structure 83 | `((either 84 | wait-for-manual-trigger 85 | wait-for-git) 86 | (with-workspace 87 | clone 88 | 89 | do-something))) 90 | ``` 91 | 92 | ### Get details on commits since last build 93 | 94 | ```clojure 95 | (defn clone [args ctx] 96 | (lambdacd-git/clone ctx repo branch-or-tag-or-commit-hash (:cwd args))) 97 | 98 | (def pipeline-structure 99 | `(wait-for-git 100 | (with-workspace 101 | clone 102 | lambdacd-git/list-changes 103 | do-something))) 104 | ``` 105 | 106 | ### Working with more than one repository 107 | 108 | You can have clone steps that clone into different subdirectories: 109 | 110 | ```clojure 111 | (defn clone-foo [args ctx] 112 | (lambdacd-git/clone ctx repo branch-or-tag-or-commit-hash (str (:cwd args) "/" "foo"))) 113 | (defn clone-bar [args ctx] 114 | (lambdacd-git/clone ctx repo branch-or-tag-or-commit-hash (str (:cwd args) "/" "bar"))) 115 | 116 | (def pipeline-structure 117 | `(; ... 118 | (with-workspace 119 | clone-foo 120 | clone-bar 121 | do-something))) 122 | ``` 123 | 124 | If you want to use this in combination with `wait-for-git`, you need to detect which commit to use. For details, see 125 | [example/multi_repo_pipeline.clj](https://github.com/flosell/lambdacd-git/blob/master/example/lambdacd_git/example/multi_repo_pipeline.clj) 126 | 127 | ### Tagging versions 128 | 129 | You can tag any revision: 130 | 131 | ```clojure 132 | (defn deploy-to-live [args ctx] 133 | ;do deploy 134 | (let [cwd (:cwd args)] 135 | (lambdacd-git/tag-version ctx cwd repo "HEAD" (str "live-" version)))) 136 | 137 | (def pipeline-structure 138 | `(; ... 139 | (with-workspace 140 | ;... 141 | deploy-to-live 142 | do-something))) 143 | ``` 144 | ## Configuration (versions >= 0.3.0) 145 | 146 | Certain values can be configured using LambdaCDs config-map. The following example shows the default: 147 | 148 | ```clojure 149 | (let [config {:git {:timeout 20 ; the timeout for remote operations in seconds 150 | :credentials-provider (CredentialsProvider/getDefault) ; the credentials-provider to use for HTTPS clones (e.g. UsernamePasswordCredentialsProvider) 151 | :ssh {:use-agent true ; whether to use an SSH agent 152 | :known-hosts-files ["~/.ssh/known_hosts" 153 | "/etc/ssh/ssh_known_hosts"] ; which known-hosts files to use for SSH connections 154 | :identity-file nil ; override the normal SSH behavior and explicitly specify a key to use 155 | :strict-host-key-checking nil}}}] ; override the normal SSH behavior and explicitly set the StrictHostKeyChecking setting. Off by default, can be set to yes,no or ask 156 | (lambdacd/assemble-pipeline pipeline-structure config)) 157 | ``` 158 | 159 | The same parameters can also be used as parameters to git operations directly, e.g. if different repositories need different credentials: 160 | 161 | ```clojure 162 | (defn clone [args ctx] 163 | (lambdacd-git/clone ctx repo branch-or-tag-or-commit-hash (:cwd args) 164 | :credentials-provider (UsernamePasswordCredentialsProvider. (System/getenv "LAMBDACD_GIT_TESTREPO_USERNAME") 165 | (System/getenv "LAMBDACD_GIT_TESTREPO_PASSWORD")))) 166 | ``` 167 | 168 | ## Configuration (versions < 0.3.0) 169 | 170 | ### SSH Configuration 171 | 172 | LambdaCD-Git honors the default [SSH Config files](https://linux.die.net/man/5/ssh_config) from `~/.ssh/config`(`/etc/ssh/ssh_config` currently not supported (see [#23](https://github.com/flosell/lambdacd-git/issues/23))). Use this to configure things like `StrictHostKeyChecking` or the `IdentityFile`. Alternatively, some options can be configured using `ssh-init`. 173 | 174 | ### Authentication 175 | 176 | #### Git over SSH 177 | 178 | LambdaCD Git automatically picks up SSH-Keys in the default locations, e.g. `~/.ssh/id_rsa`. SSH Agents are also supported. 179 | 180 | #### Git over HTTPS 181 | 182 | Authentication for HTTPS is supported using an instance of the JGit [`CredentialsProvider`](http://download.eclipse.org/jgit/site/4.1.1.201511131810-r/apidocs/org/eclipse/jgit/transport/CredentialsProvider.html). 183 | 184 | 185 | Import e.g. [`UsernamePasswordCredentialsProvider`](http://download.eclipse.org/jgit/site/4.1.1.201511131810-r/apidocs/org/eclipse/jgit/transport/UsernamePasswordCredentialsProvider.html) into your namespace: 186 | ```clojure 187 | (ns .core 188 | ;... 189 | (:import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider) 190 | ;... 191 | ) 192 | ``` 193 | 194 | Add an instance to the LambdaCD config: 195 | 196 | ```clojure 197 | (let [config {:home-dir "/some/path" 198 | :git {:credentials-provider (UsernamePasswordCredentialsProvider. "some-username" "some-password")}}] 199 | ; ... 200 | ) 201 | ``` 202 | 203 | ### Customize SSH Client 204 | 205 | Some features of lambdacd-git (ssh-agent support, extended known_hosts support) require customizations to JGits singleton 206 | SSH session factory. Call `init-ssh!` once, e.g. in your `-main` function: 207 | 208 | ```clojure 209 | (defn -main [& args] 210 | ; ... 211 | (lambdacd-git/init-ssh!) 212 | ; ... 213 | ) 214 | ``` 215 | 216 | 217 | ## Development 218 | 219 | Call `./go` 220 | 221 | ## License 222 | 223 | Copyright © 2015 Florian Sellmayr 224 | 225 | Distributed under the Apache License 2.0 226 | -------------------------------------------------------------------------------- /dev-resources/logback-silent.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /dev-resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /doc/MIGRATION_GUIDE.md: -------------------------------------------------------------------------------- 1 | # Background 2 | 3 | `lambdacd-git` is a re-implementation of LambdaCDs git-support that aims to make Git-support more robust, friendlier to 4 | contributions and more flexible in usage. 5 | 6 | This required extracting git-support into its own library and changing some of the interfaces you got used to. 7 | The following guide is intended to make this migration as painless as possible, not necessarily to give you all the power 8 | the new library provides. So after you are done migrating, consider spending a bit more time to refactor your codebase 9 | to take advantage of the new interfaces and features. 10 | 11 | # Migrating from the built-in `lambdacd.git` implementation 12 | 13 | ## `wait-for-git` 14 | 15 | The interface for this function is almost the same as before. Only the `branch` argument was removed in favor of a more 16 | general ref-filter: 17 | 18 | ```clojure 19 | ; old 20 | (lambdacd.steps.git/wait-for-git ctx "git@github.com:flosell/testrepo.git" "some-branch" :ms-between-polls 100) 21 | (lambdacd.steps.git/wait-for-git ctx "git@github.com:flosell/testrepo.git" "master" :ms-between-polls 100) 22 | 23 | ; new 24 | (lambdacd-git.core/wait-for-git ctx "git@github.com:flosell/testrepo.git" :ms-between-polls 100 :ref "refs/heads/some-branch") 25 | (lambdacd-git.core/wait-for-git ctx "git@github.com:flosell/testrepo.git" :ms-between-polls 100) ; ref defaults to "refs/heads/master" 26 | ``` 27 | 28 | ## `wait-with-details` 29 | 30 | This function has been removed as mixes too many concerns with each other. Those concerns are now separated: 31 | 32 | * waiting for a commit ([`wait-for-git`](https://github.com/flosell/lambdacd-git#waiting-for-a-commit)) 33 | * cloning a repo ([`clone`](https://github.com/flosell/lambdacd-git#cloning-a-repository)) 34 | * generating a changelog ([`list-changes`](https://github.com/flosell/lambdacd-git#get-details-on-commits-since-last-build)) 35 | 36 | TODO: add convenience example on how to get exactly the same behavior? 37 | 38 | ## `with-git`/`with-git-branch`/`checkout-and-execute` 39 | 40 | These functions have been removed as they are either redundant or mix too many concerns. 41 | Use LambdaCDs `with-workspace` together with [`clone`](https://github.com/flosell/lambdacd-git#cloning-a-repository) instead. 42 | 43 | TODO: add convenience example on how to get exactly the same behavior? -------------------------------------------------------------------------------- /example/lambdacd_git/example/multi_repo_pipeline.clj: -------------------------------------------------------------------------------- 1 | (ns lambdacd-git.example.multi-repo-pipeline 2 | (:use [compojure.core]) 3 | (:require [lambdacd.steps.shell :as shell] 4 | [lambdacd.steps.manualtrigger :refer [wait-for-manual-trigger]] 5 | [lambdacd.steps.control-flow :refer [either with-workspace in-parallel run]] 6 | [lambdacd.core :as lambdacd] 7 | [ring.server.standalone :as ring-server] 8 | [lambdacd.ui.core :as ui] 9 | [lambdacd-git.core :as core] 10 | [lambdacd.runners :as runners] 11 | [lambdacd-git.test-utils :as test-utils])) 12 | 13 | (def testrepo-remote "git@github.com:flosell/testrepo") 14 | (def git-remote "git@github.com:flosell/lambdacd-git") 15 | 16 | (defn wait-for [remote] 17 | (fn [args ctx] 18 | (core/wait-for-git ctx remote 19 | :ref "refs/heads/master" 20 | :ms-between-polls 1000))) 21 | 22 | (defn- revision-or-master [args remote] 23 | ; if a commit on this remote triggered the build, use the revision that triggered it, 24 | ; otherwise, use the head of master. wait-for-git supplies the changed remote 25 | (if (= (:changed-remote args) remote) 26 | (:revision args) 27 | "master")) 28 | 29 | (defn clone [subdir ^:hide remote] 30 | (fn [args ctx] 31 | (let [revision (revision-or-master args remote)] 32 | (core/clone ctx remote revision (str (:cwd args) "/" subdir))))) 33 | 34 | (defn list-files [args ctx] 35 | (shell/bash ctx (:cwd args) 36 | "tree -L 3")) 37 | 38 | (def pipeline-structure 39 | `((either 40 | wait-for-manual-trigger 41 | (wait-for ~git-remote) 42 | (wait-for ~testrepo-remote)) 43 | (with-workspace 44 | (in-parallel 45 | (clone "git" ~git-remote) 46 | (clone "testrepo" ~testrepo-remote)) 47 | list-files))) 48 | 49 | (defn -main [& args] 50 | (let [home-dir (test-utils/create-temp-dir) 51 | config {:home-dir home-dir} 52 | pipeline (lambdacd/assemble-pipeline pipeline-structure config)] 53 | (runners/start-one-run-after-another pipeline) 54 | (ring-server/serve (routes 55 | (ui/ui-for pipeline) 56 | (core/notifications-for pipeline)) 57 | {:open-browser? false 58 | :port 8082}))) 59 | -------------------------------------------------------------------------------- /example/lambdacd_git/example/simple_pipeline.clj: -------------------------------------------------------------------------------- 1 | (ns lambdacd-git.example.simple-pipeline 2 | (:use [compojure.core]) 3 | (:require [lambdacd.steps.shell :as shell] 4 | [lambdacd.steps.manualtrigger :refer [wait-for-manual-trigger]] 5 | [lambdacd.steps.control-flow :refer [either with-workspace in-parallel run]] 6 | [lambdacd.core :as lambdacd] 7 | [ring.server.standalone :as ring-server] 8 | [lambdacd.ui.core :as ui] 9 | [lambdacd-git.core :as core] 10 | [lambdacd.runners :as runners] 11 | [clojure.java.io :as io])) 12 | 13 | (def repo-uri "https://github.com/flosell/testrepo.git") 14 | 15 | (defn wait-for-git [args ctx] 16 | (core/wait-for-git ctx repo-uri 17 | :ref "refs/heads/master" 18 | :ms-between-polls (* 60 1000))) 19 | 20 | (defn clone [args ctx] 21 | (core/clone ctx repo-uri (:revision args) (:cwd args))) 22 | 23 | (defn ls [args ctx] 24 | (shell/bash ctx (:cwd args) "ls")) 25 | 26 | (def pipeline-structure 27 | `((either 28 | wait-for-manual-trigger 29 | wait-for-git) 30 | (with-workspace 31 | clone 32 | core/list-changes 33 | ls))) 34 | 35 | (defn -main [& args] 36 | (let [home-dir (io/file "/tmp/foo") 37 | config {:home-dir home-dir} 38 | pipeline (lambdacd/assemble-pipeline pipeline-structure config)] 39 | (runners/start-one-run-after-another pipeline) 40 | (ring-server/serve (routes 41 | (ui/ui-for pipeline) 42 | (core/notifications-for pipeline)) 43 | {:open-browser? false 44 | :port 8082}))) 45 | -------------------------------------------------------------------------------- /go: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | SILENT="true" 5 | 6 | create-testrepo() { 7 | curl -sSf -H "Private-Token: ${LAMBDACD_GIT_TESTREPO_PASSWORD}" --output /dev/null -XPOST "https://gitlab.com/api/v4/projects?name=${TESTREPO_NAME}" 8 | tmpdir=$(mktemp -d) 9 | git clone "${LAMBDACD_GIT_TESTREPO_SSH}" 10 | cd "${TESTREPO_NAME}" 11 | touch hello 12 | git add hello 13 | git commit -m "initializing" 14 | git push origin master 15 | cd - 16 | } 17 | 18 | delete-testrepo() { 19 | rm -rf "${TESTREPO_NAME}" 20 | curl -sSf -H "Private-Token: ${LAMBDACD_GIT_TESTREPO_PASSWORD}" --output /dev/null -XDELETE "https://gitlab.com/api/v4/projects/${TESTREPO_PATH}" 21 | } 22 | 23 | test() { 24 | CMD="lein" 25 | 26 | if [ "${SILENT}" == "true" ]; then 27 | CMD="${CMD} with-profile dev,silent test" 28 | else 29 | CMD="${CMD} test" 30 | fi 31 | 32 | if [ -z "${LAMBDACD_GIT_TESTREPO_USERNAME}" ] || [ -z "${LAMBDACD_GIT_TESTREPO_PASSWORD}" ]; then 33 | echo 34 | echo "================================================================================" 35 | echo "Could not find LAMBDACD_GIT_TESTREPO_USERNAME AND LAMBDACD_GIT_TESTREPO_PASSWORD" 36 | echo "SKIPPING end-to-end tests that require authentication to remote repository." 37 | echo "For Travis CI builds against Pull Requests, this is expected." 38 | echo 39 | echo "To run end to end tests against your own repository, set these variables:" 40 | echo "- LAMBDACD_GIT_TESTREPO_SSH" 41 | echo "- LAMBDACD_GIT_TESTREPO_HTTPS" 42 | echo "- LAMBDACD_GIT_TESTREPO_USERNAME" 43 | echo "- LAMBDACD_GIT_TESTREPO_PASSWORD" 44 | echo "================================================================================" 45 | echo 46 | 47 | CMD="${CMD} :skip-e2e-with-auth" 48 | fi 49 | 50 | if [ -n "${TRAVIS_JOB_NUMBER}" ]; then 51 | TESTREPO_NAME="testrepo-$(echo "${TRAVIS_JOB_NUMBER}" | sed -e 's/\./-/g')" 52 | TESTREPO_PATH="flosell-test%2F${TESTREPO_NAME}" 53 | 54 | export LAMBDACD_GIT_TESTREPO_SSH="git@gitlab.com:flosell-test/${TESTREPO_NAME}.git" 55 | export LAMBDACD_GIT_TESTREPO_HTTPS="https://gitlab.com/flosell-test/${TESTREPO_NAME}.git" 56 | 57 | trap delete-testrepo EXIT 58 | create-testrepo 59 | fi 60 | 61 | ${CMD} 62 | } 63 | 64 | push() { 65 | test && git push 66 | } 67 | 68 | release() { 69 | test && lein release && scripts/github-release.sh 70 | } 71 | 72 | clean-up-testrepo() { 73 | tmp_dir=$(mktemp -d) 74 | git clone git@gitlab.com:flosell-test/testrepo.git "${tmp_dir}" 75 | pushd "${tmp_dir}" > /dev/null 76 | git tag -l | xargs -n 50 git push --delete origin 77 | popd > /dev/null 78 | rm -rf "${tmp_dir}" 79 | } 80 | 81 | function run() { 82 | if [ -z $1 ]; then 83 | lein run 84 | else 85 | NAMESPACE="lambdacd-git.example.$1-pipeline" 86 | lein run -m ${NAMESPACE} 87 | fi 88 | } 89 | 90 | if [ $# -ne 0 ] && type $1 &>/dev/null; then 91 | $1 $2 92 | else 93 | echo "usage: $0 94 | 95 | goal: 96 | test -- run all tests 97 | clean-up-testrepo -- clean up temporary data left behind in by end-to-end tests in test-repo 98 | push -- run all tests and push current state 99 | run -- run the simple sample pipeline 100 | run multi-repo -- run the multi-repo sample pipeline 101 | release -- release current version" 102 | 103 | exit 1 104 | fi 105 | -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (def lambdacd-version (or 2 | (System/getenv "LAMBDACD_VERSION") 3 | "0.13.5")) 4 | 5 | (def clojure-version-to-use (or 6 | (System/getenv "CLOJURE_VERSION") 7 | "1.7.0")) 8 | 9 | (println "Building against LambdaCD version" lambdacd-version "and clojure" clojure-version-to-use) 10 | 11 | (defproject lambdacd-git "0.4.2-SNAPSHOT" 12 | :description "Git support for LambdaCD" 13 | :url "https://github.com/flosell/lambdacd-git" 14 | :license {:name "Apache License, version 2.0" 15 | :url "http://www.apache.org/licenses/LICENSE-2.0.html"} 16 | :dependencies [[org.clojure/clojure ~clojure-version-to-use] 17 | [org.eclipse.jgit/org.eclipse.jgit "4.1.1.201511131810-r"] 18 | [com.jcraft/jsch.agentproxy.jsch "0.0.8"] 19 | [com.jcraft/jsch.agentproxy.usocket-jna "0.0.8"] 20 | [com.jcraft/jsch.agentproxy.sshagent "0.0.8"] 21 | [me.raynes/conch "0.8.0"] 22 | [lambdacd ~lambdacd-version] 23 | [ring/ring-core "1.6.3"]] 24 | :repositories {"jgit-repository" "https://repo.eclipse.org/content/groups/releases/"} 25 | :deploy-repositories [["clojars" {:creds :gpg}] 26 | ["releases" :clojars]] 27 | :test-paths ["test" "example"] 28 | :test-selectors {:default (constantly true) 29 | :skip-e2e-with-auth (complement :e2e-with-auth)} 30 | :profiles {:dev {:main lambdacd-git.example.simple-pipeline 31 | :dependencies [[compojure "1.6.0"] 32 | [ring-server "0.4.0"] 33 | [ring/ring-mock "0.2.0"]]} 34 | :silent {:jvm-opts ["-Dlogback.configurationFile=./dev-resources/logback-silent.xml"]}}) 35 | -------------------------------------------------------------------------------- /scripts/github-release.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | # TODO: make sure the following is installed: 6 | # https://github.com/aktau/github-release 7 | # https://github.com/mtdowling/chag 8 | # $GITHUB_TOKEN is set 9 | 10 | SCRIPT_DIR=$(dirname "$0") 11 | cd ${SCRIPT_DIR}/.. 12 | 13 | VERSION=$(chag latest) 14 | CHANGELOG=$(chag contents) 15 | USER="flosell" 16 | REPO="lambdacd-git" 17 | 18 | echo "Publishing Release to GitHub: " 19 | echo "Version ${VERSION}" 20 | echo "${CHANGELOG}" 21 | echo 22 | 23 | github-release release \ 24 | --user ${USER} \ 25 | --repo ${REPO} \ 26 | --tag ${VERSION} \ 27 | --name ${VERSION} \ 28 | --description "${CHANGELOG}" 29 | 30 | echo "Published release" -------------------------------------------------------------------------------- /scripts/id_rsa_gitlab-test.enc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flosell/lambdacd-git/3677c7afc1324bef6eb1c20f66bd83d99a1c54a9/scripts/id_rsa_gitlab-test.enc -------------------------------------------------------------------------------- /scripts/travis_prebuild.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | SCRIPT_DIR=$(dirname "$0") 4 | 5 | set -e 6 | 7 | # We have tests that deal with git. They need to have this set to pass 8 | git config --global user.email "you@example.com" 9 | git config --global user.name "Your Name" 10 | 11 | # decrypt ssh key to access test-gitlab: 12 | if [ -z "${encrypted_766accd2732a_key}" ] || [ -z "${encrypted_766accd2732a_iv}" ]; then 13 | echo "SKIPPING GITLAB SSH KEY DECRYPTION, no TravisCI encryption keys found." 14 | exit 0; 15 | else 16 | mkdir -p ~/.ssh 17 | openssl aes-256-cbc -K "${encrypted_766accd2732a_key}" -iv "${encrypted_766accd2732a_iv}" -in "${SCRIPT_DIR}/id_rsa_gitlab-test.enc" -out ~/.ssh/id_rsa_gitlab-test -d 18 | cat >> ~/.ssh/config <> new-entries 24 | (first) 25 | (key)) 26 | last-seen-revision (get old-revisions changed-ref) 27 | new-revision (get new-revisions changed-ref)] 28 | {:changed-ref changed-ref 29 | :revision new-revision 30 | :old-revision last-seen-revision})) 31 | 32 | (defn- report-git-exception [ref remote e] 33 | (log/warn e (str "could not get current revision for ref " ref " on " remote)) 34 | (println "could not get current revision for ref" ref "on" remote ":" (.getMessage e))) 35 | 36 | (defn git-config [ctx custom-git-config] 37 | (merge (get-in ctx [:config :git]) 38 | custom-git-config)) 39 | 40 | (defn- current-revision-or-nil [remote ref git-config] 41 | (try 42 | (git/current-revisions remote ref git-config) 43 | (catch Exception e 44 | (report-git-exception ref remote e) 45 | nil))) 46 | 47 | (defn- found-new-commit [remote last-seen-revisions current-revisions] 48 | (let [changes (find-changed-revision last-seen-revisions current-revisions)] 49 | (println "Found new commit: " (:revision changes) "on" (:changed-ref changes)) 50 | {:status :success 51 | :changed-ref (:changed-ref changes) 52 | :changed-remote remote 53 | :revision (:revision changes) 54 | :old-revision (:old-revision changes) 55 | :all-revisions current-revisions})) 56 | 57 | (defn- report-waiting-status [ctx] 58 | (async/>!! (:result-channel ctx) [:status :waiting])) 59 | 60 | (defn- wait-for-next-poll [poll-notifications ms-between-polls kill-channel] 61 | (async/alt!! 62 | kill-channel nil 63 | poll-notifications ([_] (println "Received notification. Polling out of schedule")) 64 | (async/timeout ms-between-polls) :poll)) 65 | 66 | (defn- kill-switch->ch [ctx] 67 | (let [ch (async/chan) 68 | notifier (fn [_ _ old new] 69 | (if (and (not= old new) 70 | (= true new)) 71 | (async/put! ch :killed)))] 72 | (add-watch (:is-killed ctx) ::to-channel-watcher notifier) 73 | ch)) 74 | 75 | (defn- clean-up-kill-switch->ch [a] 76 | (remove-watch a ::to-channel-watcher)) 77 | 78 | (defn- wait-for-revision-changed [last-seen-revisions remote ref ctx ms-between-polls poll-notifications git-config] 79 | (println "Last seen revisions:" (or last-seen-revisions "None") ". Waiting for new commit...") 80 | (let [kill-channel (kill-switch->ch ctx) 81 | result (loop [last-seen-revisions last-seen-revisions] 82 | (killable/if-not-killed ctx 83 | (let [current-revisions (current-revision-or-nil remote ref git-config)] 84 | (if (and 85 | (not (nil? current-revisions)) 86 | (not= current-revisions last-seen-revisions)) 87 | (found-new-commit remote last-seen-revisions current-revisions) 88 | (do 89 | (report-waiting-status ctx) 90 | (wait-for-next-poll poll-notifications ms-between-polls kill-channel) 91 | (recur last-seen-revisions))))))] 92 | (clean-up-kill-switch->ch (:is-killed ctx)) 93 | result)) 94 | 95 | (defn- last-seen-revisions-from-history [ctx] 96 | (let [last-step-result (pipeline-state/most-recent-step-result-with :_git-last-seen-revisions ctx)] 97 | (:_git-last-seen-revisions last-step-result))) 98 | 99 | (defn- initial-revisions [ctx remote ref custom-git-config] 100 | (or 101 | (last-seen-revisions-from-history ctx) 102 | (current-revision-or-nil remote ref custom-git-config))) 103 | 104 | (defn- persist-last-seen-revisions [wait-for-result last-seen-revisions ctx] 105 | (let [current-revisions (:all-revisions wait-for-result) 106 | revisions-to-persist (or current-revisions last-seen-revisions)] 107 | (async/>!! (:result-channel ctx) [:_git-last-seen-revisions revisions-to-persist]) ; by sending it through the result-channel, we can be pretty sure users don't overwrite it 108 | (assoc wait-for-result :_git-last-seen-revisions revisions-to-persist))) 109 | 110 | (defn- regex? [x] 111 | (instance? Pattern x)) 112 | 113 | (defn- to-ref-pred [ref-spec] 114 | (cond 115 | (string? ref-spec) (git/match-ref ref-spec) 116 | (regex? ref-spec) (git/match-ref-by-regex ref-spec) 117 | :else ref-spec)) 118 | 119 | (defn only-matching-remote [remote c] 120 | (let [filtering-chan (async/chan 1 (filter #(= remote (:remote %))))] 121 | (async/pipe c filtering-chan) 122 | filtering-chan)) 123 | 124 | (defn wait-for-git 125 | "Step that waits for the head of a ref to change. 126 | 127 | Takes the steps ctx and the remotes uri and the following optional parameters: 128 | * `ms-between-polls`: The milliseconds to wait between polling for a change 129 | * `ref`: The name of a ref (e.g. of a branch or a tag) where the step waits for a change. 130 | Can also be a function-predicate that takes the name of the ref as string 131 | * Custom git-options (override the default config)" 132 | [ctx remote & {:keys [ref ms-between-polls] 133 | :as custom-git-config-and-params 134 | :or {ms-between-polls (* 10 1000) 135 | ref "refs/heads/master"}}] 136 | (output/capture-output ctx 137 | (let [ref-pred (to-ref-pred ref) 138 | git-config (git-config ctx custom-git-config-and-params) 139 | initial-revisions (initial-revisions ctx remote ref-pred git-config) 140 | remote-poll-subscription (event-bus/subscribe ctx ::git-remote-poll-notification) 141 | remote-poll-notifications (only-matching-remote remote 142 | (event-bus/only-payload remote-poll-subscription)) 143 | wait-for-result (wait-for-revision-changed initial-revisions remote ref-pred ctx ms-between-polls remote-poll-notifications git-config) 144 | result (persist-last-seen-revisions wait-for-result initial-revisions ctx)] 145 | (event-bus/unsubscribe ctx ::git-remote-poll-notification remote-poll-subscription) 146 | result))) 147 | 148 | (defn clone 149 | "Clone a repository into a given working directory. 150 | 151 | Takes the steps ctx, a repository uri and ref to clone and a working directory to clone into. 152 | Takes optional custom git-config settings." 153 | [ctx repo ref cwd & {:as custom-git-config}] 154 | (output/capture-output ctx 155 | (let [ref (or ref "master") 156 | git (git/clone-repo repo cwd (git-config ctx custom-git-config)) 157 | existing-ref (git/find-ref git ref)] 158 | (if existing-ref 159 | (do 160 | (git/checkout-ref git existing-ref) 161 | {:status :success}) 162 | (do 163 | (println "Failure: Could not find ref" ref) 164 | {:status :failure}))))) 165 | 166 | (defn iso-format [^Date date] 167 | (-> (SimpleDateFormat. "yyyy-MM-dd HH:mm:ss ZZZZ") 168 | (.format date))) 169 | 170 | (defn- print-commit [commit] 171 | (println (:hash commit) "|" (iso-format (:timestamp commit)) "|" (:author commit) "|" (:msg commit))) 172 | 173 | (defn- output-commits [commits] 174 | (doall 175 | (map print-commit commits)) 176 | {:status :success 177 | :commits commits}) 178 | 179 | (defn- failure [msg] 180 | {:status :failure :out msg}) 181 | 182 | (defn no-git-repo? [cwd] 183 | (not (.exists (io/file cwd ".git")))) 184 | 185 | (defn list-changes [args ctx] 186 | (output/capture-output ctx 187 | (let [old-revision (:old-revision args) 188 | new-revision (:revision args) 189 | cwd (:cwd args)] 190 | (cond 191 | (nil? cwd) (failure "No working directory (:cwd) defined. Did you clone the repository?") 192 | (no-git-repo? cwd) (failure "No .git directory found in working directory. Did you clone the repository?") 193 | (or (nil? old-revision) (nil? new-revision)) (do 194 | (println "No old or current revision found.") 195 | (println "Current HEAD:") 196 | (output-commits [(git/get-single-commit cwd "HEAD")])) 197 | :else (output-commits (git/commits-between cwd old-revision new-revision)))))) 198 | 199 | (defn notify-git-handler [ctx request] 200 | (let [real-req (walk/keywordize-keys (ring-params/params-request request)) 201 | remote (get-in real-req [:query-params :remote])] 202 | (if remote 203 | (do 204 | (log/debug "Notifying git about update on remote" remote) 205 | (event-bus/publish!! ctx ::git-remote-poll-notification {:remote remote}) 206 | (-> (ring-response/response "") 207 | (ring-response/status 204))) 208 | (do 209 | (log/debug "Received invalid git notification: 'remote' was missing") 210 | (-> (ring-response/response 211 | "Mandatory parameter 'remote' not found. Example: /notify-git?remote=git@github.com:flosell/testrepo") 212 | (ring-response/content-type "text/plain") 213 | (ring-response/status 400)))))) 214 | 215 | (defn notifications-for [pipeline] 216 | (compojure/POST "/notify-git" request (notify-git-handler (:context pipeline) request))) 217 | 218 | (defn ^:deprecated init-ssh! [& {:as config}] 219 | (SshSessionFactory/setInstance (ssh/session-factory-for-config config)) 220 | (reset! ssh/init-ssh-called? true)) 221 | 222 | (defn tag-version [ctx cwd repo revision tag & {:as custom-git-config}] 223 | (output/capture-output ctx 224 | (let [rev (or revision "HEAD")] 225 | (cond 226 | (nil? cwd) (failure "No working directory (:cwd) defined. Did you clone the repository?") 227 | (no-git-repo? cwd) (failure "No .git directory found in working directory. Did you clone the repository?") 228 | (or (nil? tag) (empty? tag)) (failure "No tag name was given.") 229 | (or (nil? repo) (empty? repo)) (failure "No remote repository was given.") 230 | :else (do (git/tag-revision cwd rev tag) 231 | (git/push cwd repo (git-config ctx custom-git-config)) 232 | {:status :success}))))) 233 | -------------------------------------------------------------------------------- /src/lambdacd_git/git.clj: -------------------------------------------------------------------------------- 1 | (ns lambdacd-git.git 2 | (:require [clojure.java.io :as io] 3 | [lambdacd-git.ssh :as ssh] 4 | [lambdacd-git.ssh-agent-support :as ssh-agent-support] 5 | [clojure.string :as string]) 6 | (:import (org.eclipse.jgit.api Git TransportCommand TransportConfigCallback) 7 | (org.eclipse.jgit.lib Ref TextProgressMonitor) 8 | (org.eclipse.jgit.revwalk RevCommit RevWalk) 9 | (java.util Date) 10 | (org.eclipse.jgit.transport CredentialsProvider Transport SshTransport))) 11 | 12 | (defn- ref->hash [^Ref ref] 13 | (-> ref 14 | (.getObjectId) 15 | (.name))) 16 | 17 | (defn match-ref [ref] 18 | (fn [other-ref] 19 | (= other-ref ref))) 20 | 21 | (defn match-ref-by-regex [regex] 22 | (fn [other-branch] 23 | (re-matches regex other-branch))) 24 | 25 | (defn- entry-to-ref-and-hash [entry] 26 | [(key entry) (ref->hash (val entry))]) 27 | 28 | (defn- configuration-clashes-with-init-ssh? [ssh-config] 29 | (and @ssh/init-ssh-called? 30 | (not (empty? ssh-config)))) 31 | 32 | (def ssh-config-clash-msg 33 | (string/join "\n" 34 | ["" 35 | "***** SSH CONFIGURATION CLASHES! *****" 36 | "You likely called init-ssh! and supplied ssh configuration to the config-map at the same time." 37 | "Migrate all configuration from init-ssh! to the config-map to resolve this error. See the lambdacd-git changelog for details." 38 | ""])) 39 | 40 | (defn- transport-config-callback ^TransportConfigCallback [ssh-config] 41 | (reify TransportConfigCallback 42 | (configure [this transport] 43 | (when (instance? SshTransport transport) 44 | (when (configuration-clashes-with-init-ssh? ssh-config) 45 | (throw (Exception. ssh-config-clash-msg))) 46 | (when (not @ssh/init-ssh-called?) 47 | (.setSshSessionFactory transport (ssh/session-factory-for-config ssh-config))))))) 48 | 49 | (defn- set-transport-opts [^TransportCommand transport-command {:keys [timeout ^CredentialsProvider credentials-provider ssh] 50 | :or {timeout 20 51 | credentials-provider (CredentialsProvider/getDefault) 52 | ssh {}}}] 53 | (-> transport-command 54 | (.setTimeout timeout) 55 | (.setCredentialsProvider credentials-provider) 56 | (.setTransportConfigCallback (transport-config-callback ssh)))) 57 | 58 | (defn current-revisions [remote ref-filter-pred git-config] 59 | (let [ref-map (-> (Git/lsRemoteRepository) 60 | (set-transport-opts git-config) 61 | (.setHeads true) 62 | (.setTags true) 63 | (.setRemote remote) 64 | (.callAsMap))] 65 | (->> ref-map 66 | (filter #(ref-filter-pred (key %))) 67 | (map entry-to-ref-and-hash) 68 | (into {})))) 69 | 70 | (defn resolve-object [git s] 71 | (-> git 72 | (.getRepository) 73 | (.resolve s))) 74 | 75 | (defn- ref-exists? [git ref] 76 | (-> git 77 | (resolve-object ref) 78 | (nil?) 79 | (not))) 80 | 81 | (defn- ref-or-nil [git ref] 82 | (if (ref-exists? git ref) 83 | ref 84 | nil)) 85 | 86 | (defn find-ref [git ref] 87 | (or 88 | (ref-or-nil git (str "origin/" ref)) 89 | (ref-or-nil git ref))) 90 | 91 | (defn clone-repo [repo cwd git-config] 92 | (println "Cloning" repo "...") 93 | (-> (Git/cloneRepository) 94 | (set-transport-opts git-config) 95 | (.setURI repo) 96 | (.setDirectory (io/file cwd)) 97 | (.setProgressMonitor (TextProgressMonitor. *out*)) 98 | (.call))) 99 | 100 | (defn checkout-ref [^Git git ref] 101 | (println "Checking out" ref "...") 102 | (-> git 103 | (.checkout) 104 | (.setName ref) 105 | (.call))) 106 | 107 | (defn- process-commit [^RevCommit ref] 108 | (let [hash (-> ref 109 | (.getId) 110 | (.name)) 111 | msg (-> ref 112 | (.getShortMessage)) 113 | name (-> ref 114 | (.getAuthorIdent) 115 | (.getName)) 116 | email (-> ref 117 | (.getAuthorIdent) 118 | (.getEmailAddress)) 119 | time (-> ref 120 | (.getCommitTime) 121 | (* 1000) 122 | (Date.)) 123 | author (format "%s <%s>" name email)] 124 | {:hash hash 125 | :msg msg 126 | :author author 127 | :timestamp time})) 128 | 129 | (defn- ^Git git-open [workspace] 130 | (Git/open (io/file workspace))) 131 | 132 | (defn commits-between [workspace from-hash to-hash] 133 | (let [git (git-open workspace) 134 | refs (-> git 135 | (.log) 136 | (.addRange (resolve-object git from-hash) (resolve-object git to-hash)) 137 | (.call))] 138 | (map process-commit (reverse refs)))) 139 | 140 | (defn- get-commit-reference [^Git git hash] 141 | (-> git 142 | (.getRepository) 143 | (RevWalk.) 144 | (.parseCommit (resolve-object git hash)))) 145 | 146 | (defn get-single-commit [workspace hash] 147 | (let [git (git-open workspace)] 148 | (-> git 149 | (get-commit-reference hash) 150 | (process-commit)))) 151 | 152 | (defn tag-revision [workspace hash tag] 153 | (println "Tagging " hash " with " tag "...") 154 | (let [git (git-open workspace) 155 | commit (get-commit-reference git hash)] 156 | (-> git 157 | (.tag) 158 | (.setObjectId commit) 159 | (.setName tag) 160 | (.call)))) 161 | 162 | (defn push [workspace remote git-config] 163 | (println "Pushing changes...") 164 | (let [git (git-open workspace)] 165 | (-> git 166 | (.push) 167 | (set-transport-opts git-config) 168 | (.setPushAll) 169 | (.setPushTags) 170 | (.setRemote remote) 171 | (.setProgressMonitor (TextProgressMonitor. *out*)) 172 | (.call)))) 173 | -------------------------------------------------------------------------------- /src/lambdacd_git/ssh.clj: -------------------------------------------------------------------------------- 1 | (ns lambdacd-git.ssh 2 | "Functions to customize handling of SSH connections" 3 | (:require [me.raynes.fs :as fs] 4 | [clojure.java.io :as io] 5 | [clojure.tools.logging :as log] 6 | [lambdacd-git.ssh-agent-support :as ssh-agent-support]) 7 | (:import (org.eclipse.jgit.util FS) 8 | (org.eclipse.jgit.transport JschConfigSessionFactory SshSessionFactory OpenSshConfig$Host) 9 | (java.io SequenceInputStream File FileInputStream ByteArrayInputStream) 10 | (com.jcraft.jsch JSch JSchException IdentityRepository Identity Session) 11 | (clojure.lang SeqEnumeration) 12 | (java.util Vector Collection))) 13 | 14 | (defn- empty-input-stream-as-fallback [] 15 | (ByteArrayInputStream. (byte-array 0))) 16 | 17 | (defn- known-hosts-streams [known-hosts-files] 18 | (->> known-hosts-files 19 | (map fs/expand-home) 20 | (filter fs/exists?) 21 | (map io/file) 22 | (map (fn [^File f] (FileInputStream. f))) 23 | (cons (empty-input-stream-as-fallback)) 24 | (SeqEnumeration.))) 25 | 26 | (defn set-known-hosts-customizer [known-hosts-files] 27 | (fn [^JSch jsch] 28 | (doto jsch 29 | (.setKnownHosts (SequenceInputStream. (known-hosts-streams known-hosts-files)))))) 30 | 31 | (defn set-identity-file-customizer 32 | "Explicitly set the identity file that will be used for authentication. 33 | 34 | All identities will normally be tried, this setting can allow ensuring a specific GitHub account is used 35 | with permissions to a private repo." 36 | [identity-file] 37 | (fn [^JSch jsch] 38 | (let [identity-file-full-path (str (fs/expand-home identity-file))] 39 | (try 40 | (.addIdentity jsch identity-file-full-path) 41 | (catch JSchException e 42 | (log/warn e "Problems adding custom identity file."))) 43 | (let [current (.getIdentities (.getIdentityRepository jsch))] 44 | (doto jsch 45 | (.setIdentityRepository 46 | (proxy [IdentityRepository] [] 47 | (getIdentities [] (Vector. ^Collection (filter #(= identity-file-full-path (.getName ^Identity %)) current)))))))))) 48 | 49 | (defn set-strict-host-key-checking-customizer 50 | "Explicitly set StrictHostKeyChecking parameter, overriding normal SSH configuration" 51 | [setting] 52 | (fn [^Session session] 53 | (.setConfig session "StrictHostKeyChecking" setting))) 54 | 55 | (defn session-factory 56 | "Creates a SshSessionFactory that JGit can use to create Jsch instances. 57 | Takes customizer-functions that take a Jsch instances as an argument and modify it as a side-effect" 58 | ^SshSessionFactory 59 | ([jsch-customizer-fns] ; DEPRECATAED 60 | (session-factory jsch-customizer-fns [])) 61 | ([jsch-customizer-fns session-customizer-fns] 62 | (proxy [JschConfigSessionFactory] [] 63 | (configure [^OpenSshConfig$Host host ^Session session] 64 | (doall (map #(% session) session-customizer-fns))) 65 | (createDefaultJSch [^FS fs] 66 | (let [jsch (proxy-super createDefaultJSch fs)] 67 | (doall (map #(% jsch) jsch-customizer-fns)) 68 | jsch))))) 69 | 70 | (defn session-factory-for-config [{:keys [use-agent known-hosts-files identity-file strict-host-key-checking] 71 | :or {use-agent true 72 | known-hosts-files ["~/.ssh/known_hosts" "/etc/ssh/ssh_known_hosts"]}}] 73 | (let [jsch-customizer-fns (filter some? [(when use-agent ssh-agent-support/ssh-agent-customizer) 74 | (when known-hosts-files (set-known-hosts-customizer known-hosts-files)) 75 | (when identity-file (set-identity-file-customizer identity-file))]) 76 | session-customizer-fns (filter some? [(when strict-host-key-checking (set-strict-host-key-checking-customizer strict-host-key-checking))])] 77 | (session-factory jsch-customizer-fns 78 | session-customizer-fns))) 79 | 80 | (def init-ssh-called? (atom false)) 81 | -------------------------------------------------------------------------------- /src/lambdacd_git/ssh_agent_support.clj: -------------------------------------------------------------------------------- 1 | (ns lambdacd-git.ssh-agent-support 2 | (:require [clojure.tools.logging :as log]) 3 | (:import (org.eclipse.jgit.transport SshSessionFactory JschConfigSessionFactory) 4 | (com.jcraft.jsch.agentproxy.connector SSHAgentConnector) 5 | (org.eclipse.jgit.util FS) 6 | (com.jcraft.jsch.agentproxy RemoteIdentityRepository) 7 | (com.jcraft.jsch.agentproxy.usocket JNAUSocketFactory) 8 | (com.jcraft.jsch JSch))) 9 | 10 | (defn add-ssh-agent-connector [^JSch jsch] 11 | (let [usf (JNAUSocketFactory.) 12 | con (SSHAgentConnector. usf) 13 | irepo (RemoteIdentityRepository. con)] 14 | (.setIdentityRepository jsch irepo))) 15 | 16 | (defn ssh-agent-customizer [jsch] 17 | (try 18 | (if (SSHAgentConnector/isConnectorAvailable) 19 | (add-ssh-agent-connector jsch) 20 | (log/info "No SSH-Agent connector available. SSH-Keys with passphrases will not be supported")) 21 | (catch Exception e 22 | (log/warn e "Problems with SSH Agent. Falling back to default behavior")))) 23 | -------------------------------------------------------------------------------- /test/lambdacd_git/core_test.clj: -------------------------------------------------------------------------------- 1 | (ns lambdacd-git.core-test 2 | (:require [clojure.test :refer :all :refer-macros [thrown?]] 3 | [lambdacd-git.core :refer :all :as core] 4 | [lambdacd-git.git-utils :as git-utils] 5 | [clojure.core.async :as async] 6 | [lambdacd.state.internal.pipeline-state-updater :as pipeline-state-updater] 7 | [lambdacd.state.core :as state-core] 8 | [lambdacd-git.test-utils :refer [str-containing some-ctx-with]] 9 | [lambdacd.presentation.pipeline-state :as presentation-state] 10 | [clojure.java.io :as io] 11 | [lambdacd.event-bus :as event-bus] 12 | [ring.mock.request :as ring-mock] 13 | [lambdacd-git.test-utils :as test-utils] 14 | [lambdacd.execution.core :as execution-core]) 15 | (:import (java.util.concurrent TimeoutException))) 16 | 17 | (defmacro flaky-testing [title & body] 18 | (println "*** SKIPPING FLAKY TEST:" title "***") 19 | ; do nothing with flaky tests right now 20 | ) 21 | 22 | (defn- status-updates-channel [ctx] 23 | (let [step-result-updates-ch (event-bus/only-payload 24 | (event-bus/subscribe ctx :step-result-updated)) 25 | only-status-updates (async/chan 100 (map #(get-in % [:step-result :status])))] 26 | (async/pipe step-result-updates-ch only-status-updates) 27 | only-status-updates)) 28 | 29 | (defn- clear-channel [ch] 30 | (loop [] 31 | (let [[v _] (async/alts!! [ch] :default :finished)] 32 | (when (not= :finished v) 33 | (recur))))) 34 | 35 | (defn- init-state [] 36 | (let [is-killed (atom false) 37 | ctx (some-ctx-with :is-killed is-killed) 38 | step-status-channel (status-updates-channel ctx)] 39 | (pipeline-state-updater/start-pipeline-state-updater ctx) 40 | (atom {:ctx ctx 41 | :is-killed is-killed 42 | :step-status-channel step-status-channel}))) 43 | 44 | (defn git-init [state] 45 | (swap! state #(assoc % :git (git-utils/git-init))) 46 | state) 47 | 48 | (defn set-git-remote [state remote] 49 | (swap! state #(assoc % :git {:remote remote})) 50 | state) 51 | 52 | (defn git-commit [state msg] 53 | (swap! state #(assoc % :git 54 | (git-utils/git-commit (:git %) msg))) 55 | state) 56 | 57 | (defn git-checkout-b [state new-ref] 58 | (swap! state #(assoc % :git 59 | (git-utils/git-checkout-b (:git %) new-ref))) 60 | state) 61 | 62 | (defn git-checkout [state new-ref] 63 | (swap! state #(assoc % :git 64 | (git-utils/git-checkout (:git %) new-ref))) 65 | state) 66 | 67 | (defn git-add-file [state file-name file-content] 68 | (swap! state #(assoc % :git 69 | (git-utils/git-add-file (:git %) file-name file-content))) 70 | state) 71 | 72 | (def wait-for-step-finished 10000) 73 | 74 | (defn read-channel-or-time-out [c & {:keys [timeout] 75 | :or {timeout 30000}}] 76 | (async/alt!! 77 | c ([result] result) 78 | (async/timeout timeout) (throw (Exception. "timeout!")))) 79 | 80 | (defn wait-for-step-waiting [state] 81 | (let [step-status-ch (:step-status-channel @state) 82 | result (read-channel-or-time-out 83 | (async/go 84 | (loop [] 85 | (let [status (async/")) 160 | 161 | (defn- expected-iso-timestamp [state commit-msg] 162 | (git-utils/commit-timestamp-iso (:git @state) (commit-hash-by-msg state commit-msg))) 163 | 164 | (defn- expected-timestamp [state commit-msg] 165 | (git-utils/commit-timestamp-date (:git @state) (commit-hash-by-msg state commit-msg))) 166 | 167 | (defn- trigger-notification [state & {:keys [remote-to-notify] :or { remote-to-notify nil}}] 168 | ((core/notifications-for {:context (:ctx @state)}) 169 | (ring-mock/request :post (str "/notify-git?remote="(or remote-to-notify (remote state))))) 170 | state) 171 | 172 | (defn- last-seen-revisions-from-history [ctx branch] 173 | (let [last-step-result (presentation-state/most-recent-step-result-with :_git-last-seen-revisions ctx) 174 | result (:_git-last-seen-revisions last-step-result)] 175 | (get result (str "refs/heads/"branch)))) 176 | 177 | (defn- wait-for-pipeline-state-to-have-seen-commit [state branch msg] 178 | (let [commit-hash-to-wait-for (commit-hash-by-msg state msg)] 179 | (test-utils/while-with-timeout 10000 (let [current-last-seen-hash (last-seen-revisions-from-history (:ctx @state) branch) 180 | result (not= commit-hash-to-wait-for 181 | current-last-seen-hash)] 182 | (println "waiting for" commit-hash-to-wait-for "but till now it's" current-last-seen-hash) 183 | result) 184 | (Thread/sleep 10))) 185 | state) 186 | 187 | (deftest wait-for-git-test 188 | (testing "that it waits for a new commit to happen and that it prints out information on old and new commit" 189 | (let [state (-> (init-state) 190 | (git-init) 191 | (git-commit "initial commit") 192 | (start-wait-for-git-step :ref "refs/heads/master") 193 | (git-commit "other commit") 194 | (get-step-result))] 195 | (is (= :success (:status (step-result state)))) 196 | (is (= "refs/heads/master" (:changed-ref (step-result state)))) 197 | (is (= (remote state) (:changed-remote (step-result state)))) 198 | (is (= (commit-hash-by-msg state "initial commit") (:old-revision (step-result state)))) 199 | (is (= (commit-hash-by-msg state "other commit") (:revision (step-result state)))) 200 | (is (str-containing (commit-hash-by-msg state "initial commit") (:out (step-result state)))) 201 | (is (str-containing "on refs/heads/master" (:out (step-result state)))))) 202 | (testing "that notifications on the event bus trigger polling" 203 | (let [state (-> (init-state) 204 | (git-init) 205 | (git-commit "initial commit") 206 | (start-wait-for-git-step :ref "refs/heads/master" :ms-between-polls (* 2 wait-for-step-finished)) 207 | (git-commit "other commit") 208 | (trigger-notification) 209 | (get-step-result))] 210 | (is (= :success (:status (step-result state)))) 211 | (is (= "refs/heads/master" (:changed-ref (step-result state)))) 212 | (is (= (commit-hash-by-msg state "other commit") (:revision (step-result state)))) 213 | (is (str-containing "Received notification. Polling out of schedule" (:out (step-result state)))))) 214 | (testing "that notifications on the event bus for other remotes are ignored" 215 | (let [state (-> (init-state) 216 | (git-init) 217 | (git-commit "initial commit") 218 | (start-wait-for-git-step :ref "refs/heads/master" :ms-between-polls (* 2 wait-for-step-finished)) 219 | (git-commit "other commit") 220 | (trigger-notification :remote-to-notify "some-other-remote"))] 221 | (is (thrown? Exception (wait-for-step-to-complete state :timeout 500))))) 222 | (testing "that we can pass a function to filter refs we want to react on" 223 | (let [state (-> (init-state) 224 | (git-init) 225 | (git-commit "initial commit") 226 | (git-checkout-b "some-branch") 227 | (start-wait-for-git-step :ref (fn [ref] (.endsWith ref "some-branch"))) 228 | (git-commit "other commit") 229 | (get-step-result))] 230 | (is (= :success (:status (step-result state)))) 231 | (is (= (commit-hash-by-msg state "initial commit") (:old-revision (step-result state)))) 232 | (is (= (commit-hash-by-msg state "other commit") (:revision (step-result state)))) 233 | (is (str-containing (commit-hash-by-msg state "other commit") (:out (step-result state)))))) 234 | (testing "that we can pass a regex to filter refs we want to react on" 235 | (let [state (-> (init-state) 236 | (git-init) 237 | (git-commit "initial commit") 238 | (git-checkout-b "some-branch") 239 | (start-wait-for-git-step :ref #"refs/heads/some-.*") 240 | (git-commit "other commit") 241 | (get-step-result))] 242 | (is (= :success (:status (step-result state)))) 243 | (is (= (commit-hash-by-msg state "initial commit") (:old-revision (step-result state)))) 244 | (is (= (commit-hash-by-msg state "other commit") (:revision (step-result state)))) 245 | (is (str-containing (commit-hash-by-msg state "other commit") (:out (step-result state)))))) 246 | (testing "that we can pass a function that allows all refs" 247 | (let [state (-> (init-state) 248 | (git-init) 249 | (git-commit "initial commit") 250 | (git-checkout-b "some-branch") 251 | (git-commit "some commit on branch") 252 | (git-checkout "master") 253 | 254 | (start-wait-for-git-step :ref (fn [_] true)) 255 | (git-commit "some commit on master") 256 | (wait-for-step-to-complete) 257 | (wait-for-pipeline-state-to-have-seen-commit "master" "some commit on master") 258 | 259 | (git-checkout "some-branch") 260 | (start-wait-for-git-step :ref (fn [_] true)) 261 | (git-commit "some other commit on branch") 262 | 263 | (get-step-result))] 264 | (is (= :success (:status (step-result state)))) 265 | (is (= (commit-hash-by-msg state "some commit on branch") (:old-revision (step-result state)))) 266 | (is (= (commit-hash-by-msg state "some other commit on branch") (:revision (step-result state)))) 267 | (is (str-containing (commit-hash-by-msg state "some commit on branch") (:out (step-result state)))))) 268 | (testing "that it prints out information on old and new commit hashes" 269 | (let [state (-> (init-state) 270 | (git-init) 271 | (git-commit "initial commit") 272 | (start-wait-for-git-step) 273 | (git-commit "other commit") 274 | (get-step-result))] 275 | (is (str-containing (commit-hash-by-msg state "initial commit") (:out (step-result state)))) 276 | (is (str-containing (commit-hash-by-msg state "other commit") (:out (step-result state)))))) 277 | (flaky-testing "that waiting returns immediately when a commit happened while it was not waiting" 278 | (let [state (-> (init-state) 279 | (git-init) 280 | (git-commit "initial commit") 281 | (start-wait-for-git-step) 282 | (git-commit "other commit") 283 | (wait-for-step-to-complete) 284 | (wait-for-pipeline-state-to-have-seen-commit "master" "other commit") 285 | (git-commit "commit while not waiting") 286 | (start-wait-for-git-step :wait false) 287 | (get-step-result))] 288 | (is (= :success (:status (step-result state)))) 289 | (is (= (commit-hash-by-msg state "other commit") (:old-revision (step-result state)))) 290 | (is (= (commit-hash-by-msg state "commit while not waiting") (:revision (step-result state)))))) 291 | (testing "that wait-for can be killed and that the last seen revision is being kept" 292 | (let [state (-> (init-state) 293 | (git-init) 294 | (git-commit "initial commit") 295 | (start-wait-for-git-step) 296 | (kill-waiting-step) 297 | (get-step-result))] 298 | (is (= :killed (:status (step-result state)))) 299 | (is (= {"refs/heads/master" (commit-hash-by-msg state "initial commit")} (:_git-last-seen-revisions (step-result state)))))) 300 | (testing "that wait-for can be killed quickly even if it is polling very slowly" 301 | (let [state (-> (init-state) 302 | (git-init) 303 | (git-commit "initial commit") 304 | (start-wait-for-git-step :ms-between-polls (* 60 1000)) 305 | (kill-waiting-step) 306 | (get-step-result))] 307 | (is (= :killed (:status (step-result state)))))) 308 | (testing "that it retries until being killed if the repository cannot be reached" 309 | (let [state (-> (init-state) 310 | (set-git-remote "some-uri-that-doesnt-exist") 311 | (start-wait-for-git-step) 312 | (kill-waiting-step) 313 | (get-step-result))] 314 | (is (= :killed (:status (step-result state)))))) 315 | (testing "that it prints out errors if a repository can't be reached" 316 | (let [state (-> (init-state) 317 | (set-git-remote "some-uri-that-doesnt-exist") 318 | (start-wait-for-git-step) 319 | (kill-waiting-step) 320 | (get-step-result))] 321 | (is (str-containing "some-uri-that-doesnt-exist" (:out (step-result state)))))) 322 | (testing "that it assumes master if no ref is given" 323 | (let [state (-> (init-state) 324 | (git-init) 325 | (start-wait-for-git-step) 326 | (git-commit "initial commit") 327 | (get-step-result))] 328 | (is (= :success (:status (step-result state)))) 329 | (is (= (commit-hash-by-msg state "initial commit") (:revision (step-result state)))))) 330 | (testing "that it does not overwrite the latest commit with nil if polling for a new commit fails" 331 | (let [was-called? (atom false) 332 | return-nil-on-first-call-then-some-commit (fn [_ _ _] (if @was-called? "some commit hash" (do (reset! was-called? true) nil)))] 333 | (with-redefs [core/initial-revisions (constantly "some commit hash") 334 | core/current-revision-or-nil return-nil-on-first-call-then-some-commit] 335 | (let [wait-at-least-two-polls (fn [state] (do (Thread/sleep 3) state)) 336 | state (-> (init-state) 337 | (start-wait-for-git-step :ms-between-polls 1) 338 | (wait-at-least-two-polls) 339 | (kill-waiting-step) 340 | (get-step-result))] 341 | (is (str-containing "some commit hash" (:out (step-result state))))))))) 342 | 343 | (deftest clone-test 344 | (testing "that we can clone a specific commit" 345 | (let [state (init-state) 346 | workspace (test-utils/create-temp-dir)] 347 | (-> state 348 | (git-init) 349 | (git-add-file "some-file" "some content") 350 | (git-commit "first commit") 351 | (git-add-file "some-file" "some other content") 352 | (git-commit "second commit") 353 | (start-clone-step (commit-hash-by-msg state "first commit") workspace) 354 | (get-step-result)) 355 | (is (= :success (:status (step-result state)))) 356 | (is (= "some content" 357 | (slurp (io/file workspace "some-file")))))) 358 | (testing "that it falls back to head of master if ref is nil (e.g. because manual trigger instead of wait-for-git)" 359 | (let [state (init-state) 360 | workspace (test-utils/create-temp-dir)] 361 | (-> state 362 | (git-init) 363 | (git-add-file "some-file" "some content") 364 | (git-commit "first commit") 365 | (git-add-file "some-file" "some other content") 366 | (git-commit "second commit") 367 | (start-clone-step nil workspace) 368 | (get-step-result)) 369 | (is (= :success (:status (step-result state)))) 370 | (is (= "some other content" 371 | (slurp (io/file workspace "some-file")))))) 372 | (testing "that we can get information on the progress of a clone" 373 | (let [state (init-state) 374 | workspace (test-utils/create-temp-dir)] 375 | (-> state 376 | (git-init) 377 | (git-add-file "some-file" "some content") 378 | (git-commit "some commit") 379 | (start-clone-step (commit-hash-by-msg state "some commit") workspace) 380 | (get-step-result)) 381 | (is (str-containing "Receiving" (:out (step-result state)))))) 382 | (testing "that we get a proper error if a commit cant be found" 383 | (let [state (init-state) 384 | workspace (test-utils/create-temp-dir)] 385 | (-> state 386 | (git-init) 387 | (git-add-file "some-file" "some content") 388 | (git-commit "some commit") 389 | (start-clone-step "some-branch" workspace) 390 | (get-step-result)) 391 | (is (= :failure (:status (step-result state)))) 392 | (is (str-containing "Could not find ref some-branch" (:out (step-result state))))))) 393 | 394 | (deftest list-changes-test 395 | (testing "normal behavior" 396 | (let [state (init-state) 397 | workspace (test-utils/create-temp-dir)] 398 | (-> state 399 | (git-init) 400 | (git-commit "first commit") 401 | (git-commit "second commit") 402 | (git-commit "third commit") 403 | (start-clone-step "master" workspace) 404 | (wait-for-step-to-complete) 405 | (start-list-changes-step workspace (commit-hash-by-msg state "first commit") (commit-hash-by-msg state "third commit")) 406 | (get-step-result)) 407 | (testing "that it returns the changed commits" 408 | (is (= [{:hash (commit-hash-by-msg state "second commit") 409 | :msg "second commit" 410 | :author (expected-author state) 411 | :timestamp (expected-timestamp state "second commit")} 412 | {:hash (commit-hash-by-msg state "third commit") 413 | :msg "third commit" 414 | :author (expected-author state) 415 | :timestamp (expected-timestamp state "third commit")}] (:commits (step-result state))))) 416 | (testing "that it is successful" 417 | (is (= :success 418 | (:status (step-result state))))) 419 | (testing "that it outputs the commits messages" 420 | (is (str-containing "second commit" (:out (step-result state)))) 421 | (is (str-containing "third commit" (:out (step-result state))))) 422 | (testing "that it outputs the commit hashes" 423 | (is (str-containing (commit-hash-by-msg state "second commit") (:out (step-result state)))) 424 | (is (str-containing (commit-hash-by-msg state "third commit") (:out (step-result state))))) 425 | (testing "that it outputs the authors" 426 | (is (str-containing (expected-author state) (:out (step-result state))))) 427 | (testing "that it outputs formatted commit timestamps" 428 | (is (str-containing (expected-iso-timestamp state "second commit") (:out (step-result state))))))) 429 | (testing "error handling" 430 | (testing "that an error is reported if no cwd is set" 431 | (let [state (init-state)] 432 | (-> state 433 | (start-list-changes-step nil "some hash" "some other hash") 434 | (get-step-result)) 435 | (is (str-containing "No working directory" (:out (step-result state)))) 436 | (is (= :failure (:status (step-result state)))))) 437 | (testing "that an error is reported if no git repo is found in cwd" 438 | (let [state (init-state) 439 | workspace (test-utils/create-temp-dir)] 440 | (-> state 441 | (start-list-changes-step workspace "some hash" "some other hash") 442 | (get-step-result)) 443 | (is (str-containing "No .git directory" (:out (step-result state)))) 444 | (is (= :failure (:status (step-result state)))))) 445 | (testing "that the current head commit will be reported if no old and new revisions are set" 446 | (let [state (init-state) 447 | workspace (test-utils/create-temp-dir)] 448 | (-> state 449 | (git-init) 450 | (git-commit "some commit") 451 | (start-clone-step "HEAD" workspace) 452 | (wait-for-step-to-complete) 453 | (start-list-changes-step workspace nil nil) 454 | (get-step-result)) 455 | (is (str-containing "Current HEAD" (:out (step-result state)))) 456 | (is (str-containing (commit-hash-by-msg state "some commit") (:out (step-result state)))) 457 | (is (= :success (:status (step-result state)))))))) 458 | 459 | (deftest tag-version-test 460 | (testing "normal behaviour" 461 | (testing "that it tags and pushes" 462 | (let [state (init-state) 463 | remote-git (git-utils/git-init) 464 | remote-repo (:remote remote-git)] 465 | (-> state 466 | (git-init) 467 | (git-add-file "some-file" "some content") 468 | (git-commit "some commit") 469 | (start-tag-version-step (get-in @state [:git :dir]) remote-repo (commit-hash-by-msg state "some commit") "some-tag") 470 | (get-step-result)) 471 | (let [commit (commit-hash-by-msg state "some commit")] 472 | (is (= "some-tag\n" (git-utils/git-tag-list (:git @state) commit))) 473 | (is (= "some-tag\n" (git-utils/git-tag-list remote-git commit))) 474 | (is (= :success (:status (step-result state)))))))) 475 | (testing "error handling" 476 | (testing "that an error is reported if no cwd is set" 477 | (let [state (init-state)] 478 | (-> state 479 | (start-tag-version-step nil "some-uri" "some-commit" "some-tag") 480 | (get-step-result)) 481 | (println (:out (step-result state))) 482 | (is (str-containing "No working directory" (:out (step-result state)))) 483 | (is (= :failure (:status (step-result state)))))) 484 | (testing "that an error is reported if no git repo is found in cwd" 485 | (let [state (init-state) 486 | workspace (test-utils/create-temp-dir)] 487 | (-> state 488 | (start-tag-version-step workspace "some-uri" "some-commit" "some-tag") 489 | (get-step-result)) 490 | (println (:out (step-result state))) 491 | (is (str-containing "No .git directory" (:out (step-result state)))) 492 | (is (= :failure (:status (step-result state)))))) 493 | (testing "that an error is reported if no remote repository is given" 494 | (let [state (init-state)] 495 | (-> state 496 | (git-init) 497 | (git-add-file "some-file" "some content") 498 | (git-commit "some commit") 499 | (start-tag-version-step (get-in @state [:git :dir]) "" "HEAD" "tag-name") 500 | (get-step-result)) 501 | (is (str-containing "No remote repository" (:out (step-result state)))) 502 | (is (= "" (git-utils/git-tag-list (:git @state) "HEAD"))) 503 | (is (= :failure (:status (step-result state)))))) 504 | (testing "that HEAD is tagged if no revision was given" 505 | (let [state (init-state) 506 | remote-git (git-utils/git-init) 507 | remote-repo (:remote remote-git)] 508 | (-> state 509 | (git-init) 510 | (git-add-file "some-file" "some content") 511 | (git-commit "some commit") 512 | (start-tag-version-step (get-in @state [:git :dir]) remote-repo nil "some-tag") 513 | (get-step-result)) 514 | (is (= "some-tag\n" (git-utils/git-tag-list (:git @state) "HEAD"))) 515 | (is (= "some-tag\n" (git-utils/git-tag-list remote-git "HEAD"))) 516 | (is (= :success (:status (step-result state)))))) 517 | (testing "that an error is reported if no tag name is set" 518 | (let [state (init-state) 519 | remote-git (git-utils/git-init) 520 | remote-repo (:remote remote-git)] 521 | (-> state 522 | (git-init) 523 | (git-add-file "some-file" "some content") 524 | (git-commit "some commit") 525 | (start-tag-version-step (get-in @state [:git :dir]) remote-repo "HEAD" "") 526 | (get-step-result)) 527 | (is (str-containing "No tag name" (:out (step-result state)))) 528 | (is (= "" (git-utils/git-tag-list (:git @state) "HEAD"))) 529 | (is (= :failure (:status (step-result state)))))))) 530 | -------------------------------------------------------------------------------- /test/lambdacd_git/end_to_end_test.clj: -------------------------------------------------------------------------------- 1 | (ns lambdacd-git.end-to-end-test 2 | (:require [clojure.test :refer :all] 3 | [lambdacd-git.core :as core] 4 | [lambdacd.core :as lambdacd] 5 | [lambdacd.state.core :as lambdacd-state] 6 | [lambdacd-git.example.simple-pipeline :as simple-pipeline] 7 | [lambdacd.steps.manualtrigger :as manualtrigger] 8 | [lambdacd.steps.shell :as shell] 9 | [lambdacd-git.test-utils :as test-utils] 10 | [lambdacd.steps.control-flow :refer [either with-workspace]] 11 | [lambdacd.steps.manualtrigger :refer [wait-for-manual-trigger]] 12 | [lambdacd-git.ssh :as ssh] 13 | [lambdacd.execution.core :as execution-core]) 14 | (:import (org.eclipse.jgit.transport UsernamePasswordCredentialsProvider SshSessionFactory))) 15 | 16 | (defn match-all-refs [_] 17 | true) 18 | 19 | (defn wait-for-git [args ctx] 20 | (core/wait-for-git ctx (:repo-uri (:config ctx)) 21 | :ref match-all-refs 22 | :ms-between-polls (* 1 1000))) 23 | 24 | (defn clone [args ctx] 25 | (core/clone ctx (:repo-uri (:config ctx)) (:revision args) (:cwd args))) 26 | 27 | (defn ls [args ctx] 28 | (shell/bash ctx (:cwd args) "ls")) 29 | 30 | (defn create-new-tag [args ctx] 31 | (core/tag-version ctx (:cwd args) (:repo-uri (:config ctx)) "HEAD" (str (System/currentTimeMillis)))) 32 | 33 | (def pipeline-structure 34 | `((either 35 | wait-for-manual-trigger 36 | wait-for-git) 37 | (with-workspace 38 | clone 39 | core/list-changes 40 | ls 41 | create-new-tag))) 42 | 43 | ; ====================================================================================================================== 44 | 45 | (defn trigger-id [ctx build-number step-id] 46 | (let [step-result (lambdacd-state/get-step-result ctx build-number step-id)] 47 | (:trigger-id step-result))) 48 | 49 | (defn- trigger-manually-internal [pipeline] 50 | (let [ctx (:context pipeline)] 51 | (test-utils/while-with-timeout 10000 (nil? (trigger-id ctx 1 [1 1])) 52 | (Thread/sleep 1000)) 53 | (manualtrigger/post-id ctx (trigger-id ctx 1 [1 1]) {}))) 54 | 55 | (defn- init-state [& {:keys [config pipeline-structure]}] 56 | (atom {:config config 57 | :pipeline (lambdacd/assemble-pipeline pipeline-structure config) 58 | :pipeline-structure pipeline-structure})) 59 | 60 | (defn- start-pipeline [state] 61 | (let [pipeline-structure (:pipeline-structure @state) 62 | ctx (:context (:pipeline @state))] 63 | (swap! state #(assoc % :future-pipeline-result 64 | (future 65 | (execution-core/run-pipeline pipeline-structure ctx))))) 66 | state) 67 | 68 | (defn- trigger-manually [state] 69 | (trigger-manually-internal (:pipeline @state)) 70 | state) 71 | 72 | (defn pipeline-result [state] 73 | (deref (:future-pipeline-result @state) 60000 :timeout)) 74 | 75 | (defn wait-for-completion [state] 76 | (pipeline-result state) 77 | state) 78 | 79 | (defmacro expect-status [state status] 80 | ; macro is inlined, therefore convinces cursive to mark the call location of the check as failed, not the implementation 81 | `(let [pipeline-result# (pipeline-result ~state)] 82 | (is (= ~status (:status pipeline-result#)) (str "No success in " pipeline-result#)) 83 | ~state)) 84 | 85 | (defmacro expect-success [state] 86 | `(expect-status ~state :success)) 87 | 88 | (defmacro expect-failure [state] 89 | `(expect-status ~state :failure)) 90 | 91 | (defn- init-ssh [state k v] 92 | (let [existing-session-factory (SshSessionFactory/getInstance)] 93 | (swap! state #(assoc % :existing-session-factory existing-session-factory)) 94 | (core/init-ssh! k v) 95 | state)) 96 | 97 | (defn- reset-init-ssh [state] 98 | (let [existing-session-factory (:existing-session-factory @state)] 99 | (SshSessionFactory/setInstance existing-session-factory) 100 | (swap! state #(dissoc % :existing-session-factory)) 101 | (reset! ssh/init-ssh-called? false) 102 | state)) 103 | 104 | ; ====================================================================================================================== 105 | 106 | (deftest example-pipeline-test 107 | (testing "the example-pipeline" 108 | (-> (init-state :config {:home-dir (test-utils/create-temp-dir)} 109 | :pipeline-structure simple-pipeline/pipeline-structure) 110 | (start-pipeline) 111 | (trigger-manually) 112 | (wait-for-completion) 113 | (expect-success)))) 114 | 115 | (deftest ^:e2e-with-auth end-to-end-with-auth-test 116 | (testing "a complete pipeline with all features against private repositories using https and ssh" 117 | (doseq [repo-config [{:repo-uri (or (System/getenv "LAMBDACD_GIT_TESTREPO_SSH") "git@gitlab.com:flosell-test/testrepo.git") 118 | :git {:ssh {:strict-host-key-checking "no" 119 | :identity-file "~/.ssh/id_rsa_gitlab-test" 120 | :use-agent false}}} 121 | {:repo-uri (or (System/getenv "LAMBDACD_GIT_TESTREPO_HTTPS") "https://gitlab.com/flosell-test/testrepo.git") 122 | :git {:credentials-provider (UsernamePasswordCredentialsProvider. (System/getenv "LAMBDACD_GIT_TESTREPO_USERNAME") 123 | (System/getenv "LAMBDACD_GIT_TESTREPO_PASSWORD"))}}]] 124 | (testing (:repo-uri repo-config) 125 | (-> (init-state :config (assoc repo-config :home-dir (test-utils/create-temp-dir)) 126 | :pipeline-structure pipeline-structure) 127 | (start-pipeline) 128 | (trigger-manually) 129 | (wait-for-completion) 130 | 131 | (expect-success) 132 | 133 | (start-pipeline) 134 | (wait-for-completion) 135 | 136 | (expect-success)))))) 137 | 138 | (deftest ^:e2e-with-auth backwards-compatibility-test 139 | (testing "that git operations on ssh fail if init-ssh was called and ssh config was also given" 140 | (-> (init-state :config {:repo-uri (or (System/getenv "LAMBDACD_GIT_TESTREPO_SSH") "git@gitlab.com:flosell-test/testrepo.git") 141 | :git {:ssh {:strict-host-key-checking "yes"}} 142 | :home-dir (test-utils/create-temp-dir)} 143 | :pipeline-structure pipeline-structure) 144 | (init-ssh :strict-host-key-checking "no") 145 | (start-pipeline) 146 | (trigger-manually) 147 | (wait-for-completion) 148 | 149 | (expect-failure) 150 | 151 | (reset-init-ssh))) 152 | (testing "that git operations on ssh don't fail if no ssh config was given" 153 | (-> (init-state :config {:repo-uri (or (System/getenv "LAMBDACD_GIT_TESTREPO_SSH") "git@gitlab.com:flosell-test/testrepo.git") 154 | :home-dir (test-utils/create-temp-dir)} 155 | :pipeline-structure pipeline-structure) 156 | (init-ssh :strict-host-key-checking "no") 157 | (start-pipeline) 158 | (trigger-manually) 159 | (wait-for-completion) 160 | 161 | (expect-success) 162 | 163 | (reset-init-ssh)))) 164 | -------------------------------------------------------------------------------- /test/lambdacd_git/git_test.clj: -------------------------------------------------------------------------------- 1 | (ns lambdacd-git.git-test 2 | (:require [clojure.test :refer :all] 3 | [lambdacd-git.git-utils :refer [git-init git-add-file git-commit git-checkout-b 4 | git-checkout commit-by-msg git-tag git-tag-list git-user-name 5 | git-user-email commit-timestamp-date get-last-commit-msg]] 6 | [lambdacd-git.git :refer :all] 7 | [lambdacd-git.test-utils :refer [str-containing]] 8 | [clojure.java.io :as io] 9 | [lambdacd-git.test-utils :as test-utils]) 10 | (:import (org.eclipse.jgit.api Git))) 11 | 12 | (defn git-from-dir [git-dir] 13 | (Git/open (io/file git-dir))) 14 | 15 | (defn no-branches [] 16 | (constantly false)) 17 | 18 | (defn match-branch [branch] 19 | (match-ref (str "refs/heads/" branch))) 20 | 21 | (defn match-tag [tag] 22 | (match-ref (str "refs/tags/" tag))) 23 | 24 | (defn match-all-refs [] 25 | (constantly true)) 26 | 27 | (deftest current-revision-test 28 | (testing "that it can get the head of the master branch" 29 | (let [git-handle (-> (git-init) 30 | (git-commit "some commit"))] 31 | (is (= {"refs/heads/master" (commit-by-msg git-handle "some commit")} 32 | (current-revisions (:remote git-handle) (match-branch "master") {}))))) 33 | (testing "that it can get the head of all branches" 34 | (let [git-handle (-> (git-init) 35 | (git-commit "some commit on master") 36 | (git-checkout-b "some-branch") 37 | (git-commit "some commit on branch") 38 | (git-checkout "master"))] 39 | (is (= {"refs/heads/some-branch" (commit-by-msg git-handle "some commit on branch") 40 | "refs/heads/master" (commit-by-msg git-handle "some commit on master")} 41 | (current-revisions (:remote git-handle) (match-all-refs) {}))))) 42 | (testing "that it returns an emtpy map if no ref matches" 43 | (let [git-handle (-> (git-init) 44 | (git-commit "some commit on master"))] 45 | (is (= {} 46 | (current-revisions (:remote git-handle) (no-branches) {}))))) 47 | (testing "that it can get the head of tags" 48 | (let [git-handle (-> (git-init) 49 | (git-commit "some commit on master") 50 | (git-tag "some-tag"))] 51 | (is (= {"refs/heads/master" (commit-by-msg git-handle "some commit on master") 52 | "refs/tags/some-tag" (commit-by-msg git-handle "some commit on master")} 53 | (current-revisions (:remote git-handle) (match-all-refs) {}))) 54 | (is (= {"refs/tags/some-tag" (commit-by-msg git-handle "some commit on master")} 55 | (current-revisions (:remote git-handle) (match-tag "some-tag") {})))))) 56 | 57 | (deftest clone-repo-test 58 | (testing "that we can clone the head of master" 59 | (let [git-handle (-> (git-init) 60 | (git-add-file "some-file" "some content") 61 | (git-commit "some commit on master")) 62 | workspace (test-utils/create-temp-dir)] 63 | (clone-repo (:remote git-handle) workspace {}) 64 | (is (= "some content" 65 | (slurp (io/file workspace "some-file"))))))) 66 | 67 | (deftest checkout-ref-test 68 | (testing "that we can checkout the head of a branch" 69 | (let [git-handle (-> (git-init) 70 | (git-add-file "some-file" "some content") 71 | (git-commit "some commit on master") 72 | (git-checkout-b "some-branch") 73 | (git-add-file "some-file" "some content on branch") 74 | (git-commit "some commit on branch") 75 | (git-checkout "master")) 76 | workspace (:dir git-handle)] 77 | (checkout-ref (git-from-dir workspace) "some-branch") 78 | (is (= "some content on branch" 79 | (slurp (io/file workspace "some-file")))))) 80 | (testing "that we can checkout a tag" 81 | (let [git-handle (-> (git-init) 82 | (git-add-file "some-file" "some content") 83 | (git-commit "some commit") 84 | (git-tag "some-tag") 85 | (git-add-file "some-file" "some other content") 86 | (git-commit "some other commit")) 87 | workspace (:dir git-handle)] 88 | (checkout-ref (git-from-dir workspace) "some-tag") 89 | (is (= "some content" 90 | (slurp (io/file workspace "some-file")))))) 91 | (testing "that we can checkout any commit" 92 | (let [git-handle (-> (git-init) 93 | (git-add-file "some-file" "some content") 94 | (git-commit "first commit") 95 | (git-add-file "some-file" "some other content") 96 | (git-commit "second commit")) 97 | workspace (:dir git-handle) 98 | first-commit-hash (:hash (first (:commits git-handle)))] 99 | (checkout-ref (git-from-dir workspace) first-commit-hash) 100 | (is (= "some content" 101 | (slurp (io/file workspace "some-file"))))))) 102 | 103 | (defn- expected-author [git-handle] 104 | (str (git-user-name git-handle) " <" (git-user-email git-handle) ">")) 105 | 106 | (defn expected-timestamp [git-handle commit-msg] 107 | (commit-timestamp-date git-handle (commit-by-msg git-handle commit-msg))) 108 | 109 | (deftest commits-between-test 110 | (testing "that it returns the commits between two hashes excluding the first and including the last" 111 | (let [git-handle (-> (git-init) 112 | (git-commit "first commit") 113 | (git-commit "second commit") 114 | (git-commit "third commit")) 115 | first-commit (commit-by-msg git-handle "first commit") 116 | second-commit (commit-by-msg git-handle "second commit") 117 | third-commit (commit-by-msg git-handle "third commit") 118 | workspace (:dir git-handle)] 119 | (is (= [{:hash second-commit 120 | :msg "second commit" 121 | :author (expected-author git-handle) 122 | :timestamp (expected-timestamp git-handle "second commit")} 123 | {:hash third-commit 124 | :msg "third commit" 125 | :author (expected-author git-handle) 126 | :timestamp (expected-timestamp git-handle "third commit")}] 127 | (commits-between workspace first-commit third-commit)))))) 128 | 129 | (deftest get-single-commit-test 130 | (testing "that we can get commit information for a specific commit" 131 | (let [git-handle (-> (git-init) 132 | (git-commit "some commit"))] 133 | (is (= {:hash (commit-by-msg git-handle "some commit") 134 | :msg "some commit" 135 | :author (expected-author git-handle) 136 | :timestamp (expected-timestamp git-handle "some commit")} 137 | (get-single-commit (:dir git-handle) (commit-by-msg git-handle "some commit")))))) 138 | (testing "that we can get commit information for the HEAD commit" 139 | (let [git-handle (-> (git-init) 140 | (git-commit "some commit"))] 141 | (is (= {:hash (commit-by-msg git-handle "some commit") 142 | :msg "some commit" 143 | :author (expected-author git-handle) 144 | :timestamp (expected-timestamp git-handle "some commit")} 145 | (get-single-commit (:dir git-handle) "HEAD")))))) 146 | 147 | (deftest tag-revision-test 148 | (testing "that we can tag the head of the master branch" 149 | (let [git-handle (-> (git-init) 150 | (git-commit "some commit")) 151 | workspace (:dir git-handle)] 152 | (tag-revision workspace "HEAD" "some-tag") 153 | (is (= "some-tag\n" 154 | (git-tag-list git-handle "HEAD"))))) 155 | (testing "that we can tag some commit on another branch" 156 | (let [git-handle (-> (git-init) 157 | (git-commit "some commit on master") 158 | (git-checkout-b "some-branch") 159 | (git-commit "some commit on branch") 160 | (git-commit "some other commit on branch") 161 | (git-checkout "master")) 162 | commit (commit-by-msg git-handle "some commit on branch") 163 | workspace (:dir git-handle)] 164 | (tag-revision workspace commit "some-tag") 165 | (is (= "some-tag\n" 166 | (git-tag-list git-handle commit)))))) 167 | 168 | (deftest push-test 169 | (testing "that it pushes a commit" 170 | (let [remote-git (git-init) 171 | git-handle (-> (git-init) 172 | (git-add-file "some-file" "some content") 173 | (git-commit "some commit on master")) 174 | workspace (:dir git-handle) 175 | remote (:remote remote-git)] 176 | (push workspace remote {}) 177 | (is (= "some commit on master" (clojure.string/trim (get-last-commit-msg remote-git)))))) 178 | (testing "that it pushes a tag" 179 | (let [remote-git (git-init) 180 | git-handle (-> (git-init) 181 | (git-add-file "some-file" "some content") 182 | (git-commit "some commit on master") 183 | (git-tag "some-tag")) 184 | workspace (:dir git-handle) 185 | remote (:remote remote-git)] 186 | (push workspace remote {}) 187 | (is (= "some-tag\n" (git-tag-list remote-git "HEAD")))))) 188 | -------------------------------------------------------------------------------- /test/lambdacd_git/git_utils.clj: -------------------------------------------------------------------------------- 1 | (ns lambdacd-git.git-utils 2 | (:require [me.raynes.conch :refer [let-programs]] 3 | [clojure.java.io :as io] 4 | [clojure.string :as s]) 5 | (:import (java.nio.file Files) 6 | (java.nio.file.attribute FileAttribute) 7 | (java.util Date))) 8 | 9 | 10 | 11 | (defn git [git-handle & args] 12 | (let [theargs (concat args [{:dir (:dir git-handle)}])] 13 | (let-programs [git "/usr/bin/git"] 14 | (apply git theargs)))) 15 | 16 | (defn no-file-attributes [] 17 | (into-array FileAttribute [])) 18 | 19 | (defn- create-temp-dir [] 20 | (str (Files/createTempDirectory "lambdacd-git" (no-file-attributes)))) 21 | 22 | (defn git-init [] 23 | (let [dir (create-temp-dir) 24 | git-handle {:dir dir 25 | :remote (str "file://" dir) 26 | :commits [] 27 | :commits-by-msg {} 28 | :staged-file-content nil}] 29 | (git git-handle "init") 30 | git-handle)) 31 | 32 | (defn git-add-file [git-handle file-name file-content] 33 | (spit (io/file (:dir git-handle) file-name) file-content) 34 | (git git-handle "add" "-A") 35 | (assoc git-handle :staged-file-content file-content)) 36 | 37 | (defn git-commit [git-handle msg] 38 | (git git-handle "commit" "-m" msg "--allow-empty") 39 | (let [new-hash (s/trim (git git-handle "rev-parse" "HEAD")) 40 | commit-desc {:hash new-hash :file-content (:staged-file-content git-handle)}] 41 | (-> git-handle 42 | (update :commits #(conj % commit-desc)) 43 | (update :commits-by-msg #(assoc % msg commit-desc)) 44 | (assoc :staged-file-content nil)))) 45 | 46 | (defn git-checkout-b [git-handle new-branch] 47 | (git git-handle "checkout" "-b" new-branch) 48 | git-handle) 49 | 50 | (defn git-checkout [git-handle branch] 51 | (git git-handle "checkout" branch) 52 | git-handle) 53 | 54 | (defn git-tag [git-handle tag] 55 | (git git-handle "tag" tag) 56 | git-handle) 57 | 58 | (defn git-tag-list [git-handle commit] 59 | (git git-handle "tag" "-l" "--points-at" commit)) 60 | 61 | (defn git-user-name [git-handle] 62 | (s/trim (git git-handle "config" "--get" "user.name"))) 63 | 64 | (defn git-user-email [git-handle] 65 | (s/trim (git git-handle "config" "--get" "user.email"))) 66 | 67 | (defn commit-by-msg [git-handle msg] 68 | (or 69 | (get-in git-handle [:commits-by-msg msg :hash]) 70 | (throw (Exception. (str "no hash found for " msg))))) 71 | 72 | (defn commit-timestamp-iso [git-handle hash] 73 | (git git-handle "show" "--pretty=format:%cd" "--date=iso" hash)) 74 | 75 | (defn commit-timestamp-date [git-handle hash] 76 | (let [timestamp (git git-handle "show" "--pretty=format:%ct" hash)] 77 | (-> timestamp 78 | (s/trim) 79 | (Integer/parseInt) 80 | (* 1000) 81 | (Date.)))) 82 | 83 | (defn get-last-commit-msg [git-handle] 84 | (git git-handle "log" "--pretty=%B" "-1")) -------------------------------------------------------------------------------- /test/lambdacd_git/ssh_test.clj: -------------------------------------------------------------------------------- 1 | (ns lambdacd-git.ssh-test 2 | (:require [clojure.test :refer :all] 3 | [lambdacd-git.ssh :refer :all] 4 | [me.raynes.fs :as fs]) 5 | (:import (com.jcraft.jsch JSch Session OpenSSHConfig) 6 | (org.eclipse.jgit.transport SshSessionFactory URIish CredentialsProvider OpenSshConfig$Host) 7 | (org.eclipse.jgit.util FS))) 8 | 9 | (def some-known-hosts-file-content 10 | "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==") 11 | 12 | (deftest known-hosts-customizer-test 13 | (testing "that it can read a known-hosts file" 14 | (let [known-hosts-file (fs/temp-file "known_hosts") 15 | customizer (set-known-hosts-customizer [known-hosts-file]) 16 | jsch (JSch.)] 17 | (spit known-hosts-file some-known-hosts-file-content) 18 | (customizer jsch) 19 | (is (= 1 (count (.getHostKey (.getHostKeyRepository jsch) "github.com" "ssh-rsa")))))) 20 | (testing "that it can deal with files that do not exist" 21 | (let [customizer (set-known-hosts-customizer ["i-do-not-exist"]) 22 | jsch (JSch.)] 23 | (customizer jsch) 24 | (is (= 0 (count (.getHostKey (.getHostKeyRepository jsch)))))))) 25 | 26 | 27 | (deftest session-factory-for-config-test 28 | (testing "StrictHostKeyChecking" 29 | (testing "that it sets the config on the session if requested" 30 | (let [session-factory (session-factory-for-config {:strict-host-key-checking "SomeSetting"}) 31 | session (.getSession (JSch.) "someHost")] 32 | (.configure session-factory (OpenSshConfig$Host.) session) 33 | (is (= "SomeSetting" (.getConfig session "StrictHostKeyChecking"))))))) 34 | -------------------------------------------------------------------------------- /test/lambdacd_git/test_utils.clj: -------------------------------------------------------------------------------- 1 | (ns lambdacd-git.test-utils 2 | (:require [lambdacd.internal.default-pipeline-state :as default-pipeline-state] 3 | [lambdacd.event-bus :as event-bus] 4 | [clojure.core.async :as async]) 5 | (:import (java.nio.file.attribute FileAttribute) 6 | (java.nio.file Files))) 7 | 8 | (defn- no-file-attributes [] 9 | (into-array FileAttribute [])) 10 | 11 | (def temp-prefix "lambdacd-git-test") 12 | 13 | (defn create-temp-dir [] 14 | (str (Files/createTempDirectory temp-prefix (no-file-attributes)))) 15 | 16 | (defn str-containing [expected-substring output] 17 | (.contains output expected-substring)) 18 | 19 | 20 | (defn- some-ctx-template [] 21 | (let [config {:home-dir (create-temp-dir)}] 22 | (-> {:initial-pipeline-state {} ;; only used to assemble pipeline-state, not in real life 23 | :step-id [42] 24 | :result-channel (async/chan (async/dropping-buffer 100)) 25 | :pipeline-state-component nil ;; set later 26 | :config config 27 | :is-killed (atom false) 28 | :_out-acc (atom "") 29 | :started-steps (atom #{})} 30 | (event-bus/initialize-event-bus)) 31 | )) 32 | 33 | (defn- add-pipeline-state-component [template] 34 | (if (nil? (:pipeline-state-component template)) 35 | (assoc template :pipeline-state-component 36 | (default-pipeline-state/new-default-pipeline-state (:config template) :initial-state-for-testing (:initial-pipeline-state template))) 37 | template)) 38 | 39 | (defn some-ctx-with [& args] 40 | (add-pipeline-state-component 41 | (apply assoc (some-ctx-template) args))) 42 | 43 | (defmacro while-with-timeout [timeout-ms test & body] 44 | `(let [start-timestamp# (System/currentTimeMillis)] 45 | (while (and 46 | ~test 47 | (< (System/currentTimeMillis) (+ start-timestamp# ~timeout-ms))) 48 | ~@body) 49 | (if (> (System/currentTimeMillis) (+ start-timestamp# ~timeout-ms)) 50 | (throw (Exception. "while-with-timeout timed out"))))) 51 | --------------------------------------------------------------------------------