├── subprojects ├── runner │ ├── src │ │ ├── test │ │ │ ├── groovy │ │ │ │ ├── stepdefs │ │ │ │ │ ├── .keep │ │ │ │ │ └── CliStep.groovy │ │ │ │ └── cd │ │ │ │ │ └── pipeline │ │ │ │ │ └── runner │ │ │ │ │ ├── internal │ │ │ │ │ └── DefaultServiceRegistrySpec.groovy │ │ │ │ │ └── cli │ │ │ │ │ ├── dsl │ │ │ │ │ └── PipelineScriptRunnerSpec.groovy │ │ │ │ │ └── MainSpec.groovy │ │ │ └── resources │ │ │ │ ├── helloworld.txt │ │ │ │ └── features │ │ │ │ └── commandlinehelp.feature │ │ └── main │ │ │ ├── resources │ │ │ └── META-INF │ │ │ │ └── services │ │ │ │ └── cd.pipeline.runner.internal.ServiceRegistry │ │ │ └── groovy │ │ │ └── cd │ │ │ └── pipeline │ │ │ └── runner │ │ │ ├── cli │ │ │ ├── Pipeline.groovy │ │ │ ├── dsl │ │ │ │ ├── script │ │ │ │ │ └── PipelineScript.groovy │ │ │ │ └── PipelineScriptRunner.java │ │ │ ├── command │ │ │ │ ├── MainCommand.java │ │ │ │ ├── Command.java │ │ │ │ ├── HelpCommand.java │ │ │ │ ├── FeedbackCommand.java │ │ │ │ └── RunCommand.java │ │ │ └── Main.groovy │ │ │ └── internal │ │ │ ├── ServiceRegistry.java │ │ │ └── DefaultServiceRegistry.java │ └── runner.gradle ├── api │ ├── src │ │ ├── main │ │ │ └── groovy │ │ │ │ └── cd │ │ │ │ └── pipeline │ │ │ │ ├── api │ │ │ │ ├── Task.java │ │ │ │ ├── Stage.java │ │ │ │ ├── ApiProvider.java │ │ │ │ ├── PipelineProcessor.java │ │ │ │ ├── task │ │ │ │ │ └── ShellCommand.java │ │ │ │ ├── Pipeline.java │ │ │ │ └── DefaultApiProvider.java │ │ │ │ ├── util │ │ │ │ ├── ServiceLookupException.java │ │ │ │ ├── ConfigureUtil.java │ │ │ │ └── ServiceLocator.java │ │ │ │ └── event │ │ │ │ └── PipeEvent.java │ │ └── test │ │ │ ├── resources │ │ │ └── META-INF │ │ │ │ └── services │ │ │ │ ├── cd.pipeline.util.ServiceExpectedToBeImplementedByOneClass │ │ │ │ └── cd.pipeline.util.ServiceExpectedToBeImplementedByMultipleClasses │ │ │ └── groovy │ │ │ └── cd │ │ │ └── pipeline │ │ │ └── util │ │ │ └── ServiceLocatorSpec.groovy │ └── api.gradle ├── groovydsl │ ├── src │ │ ├── main │ │ │ └── groovy │ │ │ │ └── cd │ │ │ │ └── pipeline │ │ │ │ ├── dsl │ │ │ │ ├── TaskDsl.java │ │ │ │ ├── internal │ │ │ │ │ ├── DslExporter.java │ │ │ │ │ ├── InternalTaskDsl.java │ │ │ │ │ ├── InternalStageDsl.java │ │ │ │ │ ├── InternalPipelineDsl.java │ │ │ │ │ ├── ShellCommandTaskDsl.groovy │ │ │ │ │ ├── PipelineException.java │ │ │ │ │ ├── DefaultEmailDsl.groovy │ │ │ │ │ ├── DefaultAnnounceDsl.groovy │ │ │ │ │ ├── DefaultMessengerDsl.groovy │ │ │ │ │ ├── DefaultEmailMessengerDsl.groovy │ │ │ │ │ ├── DefaultPipelineDsl.java │ │ │ │ │ └── DefaultStageDsl.java │ │ │ │ ├── MessageContextDsl.java │ │ │ │ ├── StageDsl.java │ │ │ │ ├── AnnounceDsl.java │ │ │ │ ├── MessengerDsl.java │ │ │ │ ├── EmailDsl.java │ │ │ │ ├── PipelineDsl.java │ │ │ │ └── EmailMessengerDsl.java │ │ │ │ └── messenger │ │ │ │ ├── MessageContext.java │ │ │ │ ├── SpecializedMessenger.java │ │ │ │ ├── Messenger.java │ │ │ │ └── internal │ │ │ │ ├── EmailContext.java │ │ │ │ ├── AggregateMessenger.java │ │ │ │ └── EmailMessenger.java │ │ └── test │ │ │ └── groovy │ │ │ └── cd │ │ │ └── pipeline │ │ │ ├── dsl │ │ │ └── PipelineDslSpec.groovy │ │ │ └── messenger │ │ │ └── internal │ │ │ ├── AggregateMessengerSpec.groovy │ │ │ └── EmailMessengerSpec.groovy │ └── groovydsl.gradle ├── listener │ ├── src │ │ ├── main │ │ │ ├── groovy │ │ │ │ └── cd │ │ │ │ │ └── pipeline │ │ │ │ │ └── listen │ │ │ │ │ ├── PipeListenConfiguration.groovy │ │ │ │ │ ├── core │ │ │ │ │ ├── git │ │ │ │ │ │ ├── GitUtils.groovy │ │ │ │ │ │ └── GithubUtils.groovy │ │ │ │ │ ├── GitTriggerEvent.groovy │ │ │ │ │ ├── healthcheck │ │ │ │ │ │ ├── GitCommandHealthCheck.groovy │ │ │ │ │ │ └── PipeRunnerCommandHealthCheck.groovy │ │ │ │ │ ├── DeadEventHandler.groovy │ │ │ │ │ └── GitWorker.groovy │ │ │ │ │ ├── Main.groovy │ │ │ │ │ ├── PipeListenService.groovy │ │ │ │ │ └── resources │ │ │ │ │ ├── GitLabWebHookResource.groovy │ │ │ │ │ └── GitHubWebHookResource.groovy │ │ │ └── resources │ │ │ │ └── banner.txt │ │ └── test │ │ │ ├── resources │ │ │ ├── gitlab │ │ │ │ ├── minimalPayloadMissingCommits.json │ │ │ │ ├── minimalPayloadMissingRepositoryHomepage.json │ │ │ │ ├── minimalPayloadMissingRepositoryUrl.json │ │ │ │ ├── minimalPayloadMissingPusher.json │ │ │ │ ├── minimalPayloadMissingRef.json │ │ │ │ ├── minimalPayloadMissingRepositoryName.json │ │ │ │ ├── minimalPayload.json │ │ │ │ ├── samplePayload.json │ │ │ │ └── samplePayloadFromBranch.json │ │ │ └── github │ │ │ │ ├── minimalPayloadMissingHeadCommit.json │ │ │ │ ├── minimalPayloadMissingPusher.json │ │ │ │ ├── minimalPayloadMissingRepositoryUrl.json │ │ │ │ ├── minimalPayloadMissingCommits.json │ │ │ │ ├── minimalPayloadMissingRef.json │ │ │ │ ├── minimalPayloadMissingRepositoryMasterBranch.json │ │ │ │ ├── minimalPayloadMissingRepositoryName.json │ │ │ │ ├── minimalPayloadMissingRepositoryLanguage.json │ │ │ │ ├── minimalPayload.json │ │ │ │ ├── minimalPayloadMissingRepositoryPrivateState.json │ │ │ │ ├── samplePayload.json │ │ │ │ └── samplePayloadFromBranch.json │ │ │ └── groovy │ │ │ └── cd │ │ │ └── pipeline │ │ │ └── listen │ │ │ ├── core │ │ │ ├── GitTriggerEventSpec.groovy │ │ │ ├── git │ │ │ │ ├── GitUtilsTest.groovy │ │ │ │ └── GithubUtilsSpec.groovy │ │ │ ├── GitWorkerSpec.groovy │ │ │ └── DeadEventHandlerSpec.groovy │ │ │ ├── MainSpec.groovy │ │ │ ├── PipeListenServiceSpec.groovy │ │ │ └── resources │ │ │ ├── GitLabWebHookResourceSpec.groovy │ │ │ └── GitHubWebHookResourceSpec.groovy │ ├── listener.gradle │ └── README.md ├── graphs │ ├── graphs.gradle │ └── src │ │ └── test │ │ └── groovy │ │ └── cd │ │ └── pipeline │ │ └── graphs │ │ └── GraphSpec.groovy └── configmodel │ ├── configmodel.gradle │ └── src │ └── main │ └── groovy │ └── cd │ └── pipeline │ └── config │ ├── DefaultStage.java │ └── DefaultPipeline.java ├── gradle ├── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── buildscript.gradle ├── wiki.gradle ├── requireJavaVersion7.gradle ├── cucumber.gradle └── dependencies.gradle ├── .gitignore ├── project.pipe ├── docs ├── Domain-Language-dictionary.asciidoc ├── Technical-Design-Command-Ideas.md ├── Home.md ├── User-Guide-Set-up-with-Central-Repository.md ├── User-Guide-Project-Pipeline-Configuration.md ├── Technical-Design-DSL-Design-Ideas.md ├── Vision-Why-a-new-tool-for-Continuous-Delivery.md ├── 2013-12-29-Hackaton-UI.asciidoc ├── Technical-Design-System-design.md └── Technical-Design-Braindump.textile ├── CHANGELOG.md ├── settings.gradle ├── LICENSE ├── gradlew.bat ├── README.md ├── CONTRIBUTING.md └── gradlew /subprojects/runner/src/test/groovy/stepdefs/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /subprojects/runner/src/test/resources/helloworld.txt: -------------------------------------------------------------------------------- 1 | echo "Hello World" 2 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pipelinecd/pipeline/HEAD/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /subprojects/api/src/main/groovy/cd/pipeline/api/Task.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.api; 2 | 3 | public interface Task { 4 | } 5 | -------------------------------------------------------------------------------- /subprojects/groovydsl/src/main/groovy/cd/pipeline/dsl/TaskDsl.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.dsl; 2 | 3 | public interface TaskDsl { 4 | } 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.iml 2 | *.ipr 3 | *.iws 4 | *.un~ 5 | .idea 6 | build 7 | target 8 | .gradle 9 | out 10 | classes 11 | *.sublime-* 12 | -------------------------------------------------------------------------------- /gradle/buildscript.gradle: -------------------------------------------------------------------------------- 1 | repositories { 2 | jcenter() 3 | } 4 | 5 | dependencies { 6 | classpath 'org.ajoberstar:gradle-git:0.9.0' 7 | } 8 | -------------------------------------------------------------------------------- /subprojects/api/src/test/resources/META-INF/services/cd.pipeline.util.ServiceExpectedToBeImplementedByOneClass: -------------------------------------------------------------------------------- 1 | cd.pipeline.util.SingleImplementation 2 | -------------------------------------------------------------------------------- /subprojects/api/src/main/groovy/cd/pipeline/api/Stage.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.api; 2 | 3 | public interface Stage { 4 | void add(Task task); 5 | } 6 | -------------------------------------------------------------------------------- /subprojects/runner/src/main/resources/META-INF/services/cd.pipeline.runner.internal.ServiceRegistry: -------------------------------------------------------------------------------- 1 | cd.pipeline.runner.internal.DefaultServiceRegistry 2 | -------------------------------------------------------------------------------- /subprojects/groovydsl/src/main/groovy/cd/pipeline/messenger/MessageContext.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.messenger; 2 | 3 | public interface MessageContext { 4 | 5 | } 6 | -------------------------------------------------------------------------------- /subprojects/api/src/main/groovy/cd/pipeline/api/ApiProvider.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.api; 2 | 3 | public interface ApiProvider { 4 | T get(Class clazz); 5 | } 6 | -------------------------------------------------------------------------------- /subprojects/api/src/main/groovy/cd/pipeline/util/ServiceLookupException.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.util; 2 | 3 | public class ServiceLookupException extends RuntimeException { 4 | } 5 | -------------------------------------------------------------------------------- /subprojects/api/src/main/groovy/cd/pipeline/api/PipelineProcessor.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.api; 2 | 3 | public interface PipelineProcessor { 4 | void process(Pipeline pipeline); 5 | } 6 | -------------------------------------------------------------------------------- /subprojects/groovydsl/src/main/groovy/cd/pipeline/dsl/internal/DslExporter.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.dsl.internal; 2 | 3 | public interface DslExporter { 4 | 5 | T export(); 6 | } 7 | -------------------------------------------------------------------------------- /subprojects/api/src/main/groovy/cd/pipeline/api/task/ShellCommand.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.api.task; 2 | 3 | import cd.pipeline.api.Task; 4 | 5 | public interface ShellCommand extends Task { 6 | } 7 | -------------------------------------------------------------------------------- /subprojects/runner/src/main/groovy/cd/pipeline/runner/cli/Pipeline.groovy: -------------------------------------------------------------------------------- 1 | package cd.pipeline.runner.cli 2 | 3 | def main = new Main() 4 | final exitStatus = main.run('pipe-runner', args) 5 | System.exit(exitStatus) 6 | -------------------------------------------------------------------------------- /project.pipe: -------------------------------------------------------------------------------- 1 | 2 | stage 'build', { 3 | echo 'Building...' 4 | run './gradlew check' 5 | } 6 | 7 | stage 'distribution', { 8 | echo 'Creating a distributable...' 9 | run './gradlew distTar' 10 | } 11 | -------------------------------------------------------------------------------- /subprojects/api/api.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'groovy' 2 | 3 | description = 'pipeline api --API, api, ApI, aPI' 4 | 5 | dependencies { 6 | compile libraries.groovy 7 | 8 | testCompile libraries.spock 9 | } 10 | -------------------------------------------------------------------------------- /subprojects/groovydsl/src/main/groovy/cd/pipeline/messenger/SpecializedMessenger.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.messenger; 2 | 3 | public interface SpecializedMessenger extends Messenger { 4 | boolean accepts(MessageContext context); 5 | } 6 | -------------------------------------------------------------------------------- /subprojects/api/src/main/groovy/cd/pipeline/api/Pipeline.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.api; 2 | 3 | import java.util.Set; 4 | 5 | public interface Pipeline { 6 | 7 | void add(Stage stage); 8 | 9 | Set getStages(); 10 | } 11 | -------------------------------------------------------------------------------- /subprojects/listener/src/main/groovy/cd/pipeline/listen/PipeListenConfiguration.groovy: -------------------------------------------------------------------------------- 1 | package cd.pipeline.listen 2 | 3 | import com.yammer.dropwizard.config.Configuration 4 | 5 | class PipeListenConfiguration extends Configuration { 6 | } 7 | -------------------------------------------------------------------------------- /subprojects/groovydsl/src/main/groovy/cd/pipeline/dsl/MessageContextDsl.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.dsl; 2 | 3 | import cd.pipeline.messenger.MessageContext; 4 | 5 | public interface MessageContextDsl { 6 | T toContext(); 7 | } 8 | -------------------------------------------------------------------------------- /subprojects/groovydsl/src/main/groovy/cd/pipeline/messenger/Messenger.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.messenger; 2 | 3 | import cd.pipeline.event.PipeEvent; 4 | 5 | public interface Messenger { 6 | void process(MessageContext context, PipeEvent event); 7 | } 8 | -------------------------------------------------------------------------------- /subprojects/listener/src/main/groovy/cd/pipeline/listen/core/git/GitUtils.groovy: -------------------------------------------------------------------------------- 1 | package cd.pipeline.listen.core.git 2 | 3 | class GitUtils { 4 | 5 | static String toBranchName(String gitRef) { 6 | return gitRef.split('/', 3).last() 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /subprojects/api/src/test/resources/META-INF/services/cd.pipeline.util.ServiceExpectedToBeImplementedByMultipleClasses: -------------------------------------------------------------------------------- 1 | cd.pipeline.util.MultipleImplementationsFirstImpl 2 | cd.pipeline.util.MultipleImplementationsSecondImpl 3 | cd.pipeline.util.MultipleImplementationsThirdImpl 4 | -------------------------------------------------------------------------------- /subprojects/groovydsl/src/main/groovy/cd/pipeline/dsl/internal/InternalTaskDsl.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.dsl.internal; 2 | 3 | import cd.pipeline.api.Task; 4 | import cd.pipeline.dsl.TaskDsl; 5 | 6 | public interface InternalTaskDsl extends TaskDsl, DslExporter { 7 | } 8 | -------------------------------------------------------------------------------- /subprojects/groovydsl/src/main/groovy/cd/pipeline/dsl/internal/InternalStageDsl.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.dsl.internal; 2 | 3 | import cd.pipeline.api.Stage; 4 | import cd.pipeline.dsl.StageDsl; 5 | 6 | public interface InternalStageDsl extends StageDsl, DslExporter { 7 | } 8 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Wed Aug 14 00:59:45 CEST 2013 2 | distributionBase=GRADLE_USER_HOME 3 | distributionPath=wrapper/dists 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | distributionUrl=http\://services.gradle.org/distributions/gradle-2.0-all.zip 7 | -------------------------------------------------------------------------------- /docs/Domain-Language-dictionary.asciidoc: -------------------------------------------------------------------------------- 1 | Dictionary of the deployment pipeline, continuous integration, continuous delivery, continuous deployment domain specific language. 2 | 3 | - Pipeline: 4 | ... 5 | - Stage: 6 | ... 7 | - Task: 8 | ... 9 | - Environment/Resource/...: 10 | ... -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # v0.2.0 2 | - GitLab webhook support for public and private git repostories 3 | 4 | # v0.1.0 5 | - Git support 6 | - Github webhook support for public and private git repostories 7 | - Email notification support 8 | - Configuration via Pipeline DSL (`project.pipe`) file 9 | 10 | -------------------------------------------------------------------------------- /subprojects/groovydsl/src/main/groovy/cd/pipeline/dsl/internal/InternalPipelineDsl.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.dsl.internal; 2 | 3 | import cd.pipeline.api.Pipeline; 4 | import cd.pipeline.dsl.PipelineDsl; 5 | 6 | public interface InternalPipelineDsl extends PipelineDsl, DslExporter { 7 | } 8 | -------------------------------------------------------------------------------- /subprojects/graphs/graphs.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'groovy' 2 | 3 | description = 'pipeline configuration model --the INTERNAL model implementation for the :api' 4 | 5 | dependencies { 6 | compile libraries.groovy 7 | compile project(':api') 8 | 9 | testCompile libraries.spock 10 | } 11 | -------------------------------------------------------------------------------- /subprojects/configmodel/configmodel.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'groovy' 2 | 3 | description = 'pipeline configuration model --the INTERNAL model implementation for the :api' 4 | 5 | dependencies { 6 | compile libraries.groovy 7 | compile project(':api') 8 | 9 | testCompile libraries.spock 10 | } 11 | -------------------------------------------------------------------------------- /subprojects/configmodel/src/main/groovy/cd/pipeline/config/DefaultStage.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.config; 2 | 3 | import cd.pipeline.api.Stage; 4 | import cd.pipeline.api.Task; 5 | 6 | public class DefaultStage implements Stage { 7 | 8 | @Override 9 | public void add(Task task) { 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /gradle/wiki.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'github-pages' 2 | 3 | githubPages { 4 | repoUri = 'git@github.com:pipelinecd/pipeline.wiki.git' 5 | targetBranch = 'master' 6 | workingPath = "$buildDir/wiki" 7 | commitMessage 'Publish gitHub wiki from docs' 8 | pages { 9 | from 'docs' 10 | } 11 | } -------------------------------------------------------------------------------- /subprojects/groovydsl/src/main/groovy/cd/pipeline/dsl/StageDsl.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.dsl; 2 | 3 | public interface StageDsl { 4 | 5 | String getName(); 6 | 7 | String getDescription(); 8 | 9 | void setDescription(String description); 10 | 11 | void run(String command) throws Exception; 12 | } 13 | -------------------------------------------------------------------------------- /subprojects/listener/src/test/resources/gitlab/minimalPayloadMissingCommits.json: -------------------------------------------------------------------------------- 1 | { 2 | "ref": "refs/heads/master", 3 | "repository": { 4 | "homepage": "https://git.domain.com/group/project", 5 | "name": "project", 6 | "url": "git@git.domain.com:group/project.git" 7 | }, 8 | "user_name": "Garen Torikian" 9 | } -------------------------------------------------------------------------------- /subprojects/api/src/main/groovy/cd/pipeline/api/DefaultApiProvider.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.api; 2 | 3 | import cd.pipeline.util.ServiceLocator; 4 | 5 | public class DefaultApiProvider implements ApiProvider { 6 | @Override 7 | public T get(Class clazz) { 8 | return new ServiceLocator().find(clazz); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /subprojects/listener/src/main/groovy/cd/pipeline/listen/core/GitTriggerEvent.groovy: -------------------------------------------------------------------------------- 1 | package cd.pipeline.listen.core 2 | 3 | class GitTriggerEvent { 4 | 5 | final String url 6 | final String branch 7 | 8 | GitTriggerEvent(String url, String branch) { 9 | this.branch = branch 10 | this.url = url 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /subprojects/listener/src/test/resources/gitlab/minimalPayloadMissingRepositoryHomepage.json: -------------------------------------------------------------------------------- 1 | { 2 | "commits": [ 3 | { } 4 | ], 5 | "ref": "refs/heads/master", 6 | "repository": { 7 | "name": "project", 8 | "url": "git@git.domain.com:group/project.git" 9 | }, 10 | "user_name": "Garen Torikian" 11 | } -------------------------------------------------------------------------------- /subprojects/listener/src/test/resources/gitlab/minimalPayloadMissingRepositoryUrl.json: -------------------------------------------------------------------------------- 1 | { 2 | "commits": [ 3 | { } 4 | ], 5 | "ref": "refs/heads/master", 6 | "repository": { 7 | "homepage": "https://git.domain.com/group/project", 8 | "name": "project" 9 | }, 10 | "user_name": "Garen Torikian" 11 | } -------------------------------------------------------------------------------- /subprojects/groovydsl/src/main/groovy/cd/pipeline/dsl/AnnounceDsl.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.dsl; 2 | 3 | import cd.pipeline.messenger.MessageContext; 4 | import groovy.lang.Closure; 5 | 6 | import java.util.List; 7 | 8 | public interface AnnounceDsl { 9 | void email(Closure config); 10 | 11 | List toContexts(); 12 | } 13 | -------------------------------------------------------------------------------- /subprojects/listener/src/test/resources/gitlab/minimalPayloadMissingPusher.json: -------------------------------------------------------------------------------- 1 | { 2 | "commits": [ 3 | { } 4 | ], 5 | "ref": "refs/heads/master", 6 | "repository": { 7 | "homepage": "https://git.domain.com/group/project", 8 | "name": "project", 9 | "url": "git@git.domain.com:group/project.git" 10 | } 11 | } -------------------------------------------------------------------------------- /subprojects/listener/src/test/resources/gitlab/minimalPayloadMissingRef.json: -------------------------------------------------------------------------------- 1 | { 2 | "commits": [ 3 | { } 4 | ], 5 | "repository": { 6 | "homepage": "https://git.domain.com/group/project", 7 | "name": "project", 8 | "url": "git@git.domain.com:group/project.git" 9 | }, 10 | "user_name": "Garen Torikian" 11 | } -------------------------------------------------------------------------------- /subprojects/groovydsl/src/main/groovy/cd/pipeline/dsl/MessengerDsl.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.dsl; 2 | 3 | import cd.pipeline.messenger.Messenger; 4 | import groovy.lang.Closure; 5 | import groovy.lang.DelegatesTo; 6 | 7 | public interface MessengerDsl { 8 | void email(@DelegatesTo(EmailDsl.class) Closure config); 9 | 10 | Messenger toMessenger(); 11 | } 12 | -------------------------------------------------------------------------------- /subprojects/listener/src/main/groovy/cd/pipeline/listen/Main.groovy: -------------------------------------------------------------------------------- 1 | package cd.pipeline.listen 2 | 3 | import com.yammer.dropwizard.Service 4 | 5 | class Main { 6 | 7 | static void main(String... args) { 8 | new Main().createService().run(args) 9 | } 10 | 11 | Service createService() { 12 | new PipeListenService() 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /subprojects/listener/src/main/resources/banner.txt: -------------------------------------------------------------------------------- 1 | 2 | ___ ( ) ___ ___ // ( ) ___ __ ___ ___ __ 3 | // ) ) / / // ) ) //___) ) ____ // / / (( ) ) / / //___) ) // ) ) 4 | //___/ / / / //___/ / // // / / \ \ / / // // / / 5 | // / / // ((____ // / / // ) ) / / ((____ // / / 6 | -------------------------------------------------------------------------------- /subprojects/listener/src/test/resources/gitlab/minimalPayloadMissingRepositoryName.json: -------------------------------------------------------------------------------- 1 | { 2 | "commits": [ 3 | { } 4 | ], 5 | "ref": "refs/heads/master", 6 | "repository": { 7 | "homepage": "https://git.domain.com/group/project", 8 | "url": "git@git.domain.com:group/project.git" 9 | }, 10 | "user_name": "Garen Torikian" 11 | } -------------------------------------------------------------------------------- /subprojects/listener/src/test/resources/gitlab/minimalPayload.json: -------------------------------------------------------------------------------- 1 | { 2 | "commits": [ 3 | { } 4 | ], 5 | "ref": "refs/heads/master", 6 | "repository": { 7 | "homepage": "https://git.domain.com/group/project", 8 | "name": "project", 9 | "url": "git@git.domain.com:group/project.git" 10 | }, 11 | "user_name": "Garen Torikian" 12 | } -------------------------------------------------------------------------------- /subprojects/runner/src/main/groovy/cd/pipeline/runner/cli/dsl/script/PipelineScript.groovy: -------------------------------------------------------------------------------- 1 | package cd.pipeline.runner.cli.dsl.script 2 | 3 | import cd.pipeline.dsl.PipelineDsl 4 | 5 | public abstract class PipelineScript extends Script { 6 | @Delegate 7 | private PipelineDsl pipeline; 8 | 9 | public void init(final PipelineDsl pipeline) { 10 | this.pipeline = pipeline; 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /subprojects/groovydsl/src/main/groovy/cd/pipeline/dsl/internal/ShellCommandTaskDsl.groovy: -------------------------------------------------------------------------------- 1 | package cd.pipeline.dsl.internal 2 | 3 | import cd.pipeline.api.task.ShellCommand 4 | 5 | class ShellCommandTaskDsl implements InternalTaskDsl { 6 | private String command; 7 | 8 | ShellCommandTaskDsl(String command) { 9 | this.command = command 10 | } 11 | 12 | @Override 13 | ShellCommand export() { 14 | return null 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /subprojects/groovydsl/groovydsl.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'groovy' 2 | 3 | description = 'pipeline DSL in groovy --the DSL used by users to configure their pipelines' 4 | 5 | dependencies { 6 | compile libraries.groovy 7 | compile project(':api') 8 | compile project(':configmodel') 9 | 10 | compile libraries.mavenSharedUtils 11 | compile libraries.mail 12 | compile libraries.springContextSupport 13 | 14 | testCompile libraries.spock 15 | } 16 | -------------------------------------------------------------------------------- /subprojects/api/src/main/groovy/cd/pipeline/util/ConfigureUtil.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.util; 2 | 3 | import groovy.lang.Closure; 4 | 5 | public class ConfigureUtil { 6 | private ConfigureUtil() { 7 | throw new AssertionError("Cannot instantiate this class."); 8 | } 9 | 10 | public static void configure(Object obj, Closure config) { 11 | config.setDelegate(obj); 12 | config.setResolveStrategy(Closure.DELEGATE_FIRST); 13 | config.call(); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /subprojects/listener/src/test/groovy/cd/pipeline/listen/core/GitTriggerEventSpec.groovy: -------------------------------------------------------------------------------- 1 | package cd.pipeline.listen.core 2 | 3 | import spock.lang.Specification 4 | 5 | class GitTriggerEventSpec extends Specification { 6 | 7 | def 'Event requires an url and branch'() { 8 | def url = "url" 9 | def branch = "branch" 10 | 11 | expect: 12 | def event = new GitTriggerEvent(url, branch) 13 | event.url == url 14 | event.branch == branch 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /subprojects/runner/src/main/groovy/cd/pipeline/runner/cli/command/MainCommand.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.runner.cli.command; 2 | 3 | import com.beust.jcommander.Parameter; 4 | 5 | public class MainCommand { 6 | 7 | @Parameter( 8 | names = { 9 | "--help" 10 | , "-h" 11 | } 12 | , help = true 13 | ) 14 | private boolean help = false; 15 | 16 | public boolean isHelp() { 17 | return help; 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /subprojects/listener/src/test/resources/github/minimalPayloadMissingHeadCommit.json: -------------------------------------------------------------------------------- 1 | { 2 | "commits": [ 3 | { } 4 | ], 5 | "pusher": { 6 | "email": "lolwut@noway.biz", 7 | "name": "Garen Torikian" 8 | }, 9 | "ref": "refs/heads/master", 10 | "repository": { 11 | "name": "testing", 12 | "master_branch": "master", 13 | "language": "Ruby", 14 | "private": false, 15 | "url": "https://github.com/octokitty/testing" 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /subprojects/listener/listener.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'groovy' 2 | apply plugin: 'application' 3 | 4 | applicationName = 'pipe-listen' 5 | description = 'pipe-listen --the listener for pipeline triggers' 6 | mainClassName = 'cd.pipeline.listen.Main' 7 | 8 | dependencies { 9 | compile libraries.dropwizard 10 | compile libraries.groovy 11 | compile libraries.groovyJson 12 | 13 | testCompile libraries.spock 14 | testCompile libraries.dropwizardTesting 15 | } 16 | 17 | run { 18 | args 'server' 19 | } 20 | -------------------------------------------------------------------------------- /subprojects/groovydsl/src/main/groovy/cd/pipeline/dsl/EmailDsl.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.dsl; 2 | 3 | import cd.pipeline.messenger.internal.EmailContext; 4 | 5 | import java.util.List; 6 | 7 | public interface EmailDsl extends MessageContextDsl { 8 | List getTo(); 9 | 10 | void to(String... to); 11 | 12 | void setTo(List to); 13 | 14 | List getCc(); 15 | 16 | void cc(String... cc); 17 | 18 | void setCc(List cc); 19 | 20 | EmailContext toContext(); 21 | } 22 | -------------------------------------------------------------------------------- /subprojects/groovydsl/src/main/groovy/cd/pipeline/dsl/PipelineDsl.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.dsl; 2 | 3 | import groovy.lang.Closure; 4 | import groovy.lang.DelegatesTo; 5 | 6 | public interface PipelineDsl { 7 | StageDsl stage(String name, Closure closure); 8 | 9 | void echo(Object value); 10 | 11 | void echo(String format, Object... values); 12 | 13 | MessengerDsl messenger(@DelegatesTo(MessengerDsl.class) Closure closure); 14 | 15 | AnnounceDsl announce(@DelegatesTo(AnnounceDsl.class) Closure closure); 16 | } 17 | -------------------------------------------------------------------------------- /subprojects/listener/src/test/resources/github/minimalPayloadMissingPusher.json: -------------------------------------------------------------------------------- 1 | { 2 | "commits": [ 3 | { } 4 | ], 5 | "head_commit": { 6 | "url": "https://github.com/octokitty/testing/commit/1481a2de7b2a7d02428ad93446ab166be7793fbb" 7 | }, 8 | "ref": "refs/heads/master", 9 | "repository": { 10 | "name": "testing", 11 | "master_branch": "master", 12 | "language": "Ruby", 13 | "private": false, 14 | "url": "https://github.com/octokitty/testing" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /subprojects/runner/src/test/resources/features/commandlinehelp.feature: -------------------------------------------------------------------------------- 1 | #language: en 2 | @cli 3 | Feature: Command line help 4 | As an user 5 | I want to be able to get help information on the command line 6 | so I know what options are available 7 | 8 | Scenario Outline: Provide help 9 | Given the command line application 10 | When I provide as parameter 11 | Then I expect to see the application help 12 | 13 | Examples: Using 14 | | param | 15 | | --help | 16 | | -h | 17 | | | 18 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | include 'api' 2 | include 'configmodel' 3 | include 'graphs' 4 | include 'groovydsl' 5 | include 'runner' 6 | include 'listener' 7 | 8 | rootProject.name = 'pipeline' 9 | rootProject.children.each { project -> 10 | String fileBaseName = project.name 11 | String projectDirName = "subprojects/$fileBaseName" 12 | project.projectDir = new File(settingsDir, projectDirName) 13 | project.buildFileName = "${fileBaseName}.gradle" 14 | assert project.projectDir.isDirectory() 15 | assert project.buildFile.isFile() 16 | } 17 | -------------------------------------------------------------------------------- /subprojects/groovydsl/src/main/groovy/cd/pipeline/dsl/internal/PipelineException.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.dsl.internal; 2 | 3 | public class PipelineException extends RuntimeException { 4 | public PipelineException() { 5 | super(); 6 | } 7 | 8 | public PipelineException(String message) { 9 | super(message); 10 | } 11 | 12 | public PipelineException(String message, Throwable cause) { 13 | super(message, cause); 14 | } 15 | 16 | public PipelineException(Throwable cause) { 17 | super(cause); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /subprojects/listener/src/test/groovy/cd/pipeline/listen/core/git/GitUtilsTest.groovy: -------------------------------------------------------------------------------- 1 | package cd.pipeline.listen.core.git 2 | 3 | import spock.lang.Specification 4 | import spock.lang.Unroll 5 | 6 | class GitUtilsTest extends Specification { 7 | 8 | @Unroll 9 | def "Get branchname '#branch' from gitref '#ref'"() { 10 | expect: 11 | GitUtils.toBranchName(ref) == branch 12 | 13 | where: 14 | ref || branch 15 | 'refs/heads/somebranch' || 'somebranch' 16 | '+refs/pull/42/merge' || '42/merge' 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /subprojects/configmodel/src/main/groovy/cd/pipeline/config/DefaultPipeline.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.config; 2 | 3 | import cd.pipeline.api.Pipeline; 4 | import cd.pipeline.api.Stage; 5 | 6 | import java.util.LinkedHashSet; 7 | import java.util.Set; 8 | 9 | public class DefaultPipeline implements Pipeline { 10 | 11 | private Set stages = new LinkedHashSet<>(); 12 | 13 | @Override 14 | public void add(final Stage stage) { 15 | stages.add(stage); 16 | } 17 | 18 | @Override 19 | public Set getStages() { 20 | return stages; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /subprojects/listener/src/test/resources/github/minimalPayloadMissingRepositoryUrl.json: -------------------------------------------------------------------------------- 1 | { 2 | "commits": [ 3 | { } 4 | ], 5 | "head_commit": { 6 | "url": "https://github.com/octokitty/testing/commit/1481a2de7b2a7d02428ad93446ab166be7793fbb" 7 | }, 8 | "pusher": { 9 | "email": "lolwut@noway.biz", 10 | "name": "Garen Torikian" 11 | }, 12 | "ref": "refs/heads/master", 13 | "repository": { 14 | "name": "testing", 15 | "master_branch": "master", 16 | "language": "Ruby", 17 | "private": false 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /subprojects/listener/src/test/resources/github/minimalPayloadMissingCommits.json: -------------------------------------------------------------------------------- 1 | { 2 | "head_commit": { 3 | "url": "https://github.com/octokitty/testing/commit/1481a2de7b2a7d02428ad93446ab166be7793fbb" 4 | }, 5 | "pusher": { 6 | "email": "lolwut@noway.biz", 7 | "name": "Garen Torikian" 8 | }, 9 | "ref": "refs/heads/master", 10 | "repository": { 11 | "name": "testing", 12 | "master_branch": "master", 13 | "language": "Ruby", 14 | "private": false, 15 | "url": "https://github.com/octokitty/testing" 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /subprojects/listener/src/test/resources/github/minimalPayloadMissingRef.json: -------------------------------------------------------------------------------- 1 | { 2 | "commits": [ 3 | { } 4 | ], 5 | "head_commit": { 6 | "url": "https://github.com/octokitty/testing/commit/1481a2de7b2a7d02428ad93446ab166be7793fbb" 7 | }, 8 | "pusher": { 9 | "email": "lolwut@noway.biz", 10 | "name": "Garen Torikian" 11 | }, 12 | "repository": { 13 | "name": "testing", 14 | "master_branch": "master", 15 | "language": "Ruby", 16 | "private": false, 17 | "url": "https://github.com/octokitty/testing" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /subprojects/groovydsl/src/main/groovy/cd/pipeline/dsl/internal/DefaultEmailDsl.groovy: -------------------------------------------------------------------------------- 1 | package cd.pipeline.dsl.internal 2 | 3 | import cd.pipeline.dsl.EmailDsl 4 | import cd.pipeline.messenger.internal.EmailContext 5 | 6 | class DefaultEmailDsl implements EmailDsl { 7 | List to = [] 8 | List cc = [] 9 | 10 | @Override 11 | void to(String... to) { 12 | this.to.addAll(to) 13 | } 14 | 15 | @Override 16 | void cc(String... cc) { 17 | this.cc.addAll(cc) 18 | } 19 | 20 | EmailContext toContext() { 21 | return new EmailContext(to, cc) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /subprojects/listener/src/test/resources/github/minimalPayloadMissingRepositoryMasterBranch.json: -------------------------------------------------------------------------------- 1 | { 2 | "commits": [ 3 | { } 4 | ], 5 | "head_commit": { 6 | "url": "https://github.com/octokitty/testing/commit/1481a2de7b2a7d02428ad93446ab166be7793fbb" 7 | }, 8 | "pusher": { 9 | "email": "lolwut@noway.biz", 10 | "name": "Garen Torikian" 11 | }, 12 | "ref": "refs/heads/master", 13 | "repository": { 14 | "name": "testing", 15 | "language": "Ruby", 16 | "private": false, 17 | "url": "https://github.com/octokitty/testing" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /subprojects/listener/src/test/resources/github/minimalPayloadMissingRepositoryName.json: -------------------------------------------------------------------------------- 1 | { 2 | "commits": [ 3 | { } 4 | ], 5 | "head_commit": { 6 | "url": "https://github.com/octokitty/testing/commit/1481a2de7b2a7d02428ad93446ab166be7793fbb" 7 | }, 8 | "pusher": { 9 | "email": "lolwut@noway.biz", 10 | "name": "Garen Torikian" 11 | }, 12 | "ref": "refs/heads/master", 13 | "repository": { 14 | "master_branch": "master", 15 | "language": "Ruby", 16 | "private": false, 17 | "url": "https://github.com/octokitty/testing" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /subprojects/runner/runner.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'groovy' 2 | apply plugin: 'application' 3 | apply from: script('cucumber') 4 | 5 | applicationName = 'pipe-runner' 6 | description = 'pipe-runner --the pipeline runner, executes the project pipeline a pipeline configuration file' 7 | mainClassName = 'cd.pipeline.runner.cli.Pipeline' 8 | 9 | dependencies { 10 | compile libraries.groovy 11 | compile project(':api') 12 | compile project(':groovydsl') 13 | compile project(':configmodel') 14 | compile libraries.jcommander 15 | 16 | testCompile libraries.spock 17 | testCompile libraries.hamcrest 18 | } 19 | -------------------------------------------------------------------------------- /subprojects/listener/src/test/resources/github/minimalPayloadMissingRepositoryLanguage.json: -------------------------------------------------------------------------------- 1 | { 2 | "commits": [ 3 | { } 4 | ], 5 | "head_commit": { 6 | "url": "https://github.com/octokitty/testing/commit/1481a2de7b2a7d02428ad93446ab166be7793fbb" 7 | }, 8 | "pusher": { 9 | "email": "lolwut@noway.biz", 10 | "name": "Garen Torikian" 11 | }, 12 | "ref": "refs/heads/master", 13 | "repository": { 14 | "name": "testing", 15 | "master_branch": "master", 16 | "private": false, 17 | "url": "https://github.com/octokitty/testing" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /subprojects/listener/src/test/resources/github/minimalPayload.json: -------------------------------------------------------------------------------- 1 | { 2 | "commits": [ 3 | { } 4 | ], 5 | "head_commit": { 6 | "url": "https://github.com/octokitty/testing/commit/1481a2de7b2a7d02428ad93446ab166be7793fbb" 7 | }, 8 | "pusher": { 9 | "email": "lolwut@noway.biz", 10 | "name": "Garen Torikian" 11 | }, 12 | "ref": "refs/heads/master", 13 | "repository": { 14 | "name": "testing", 15 | "master_branch": "master", 16 | "language": "Ruby", 17 | "private": false, 18 | "url": "https://github.com/octokitty/testing" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /subprojects/listener/src/test/resources/github/minimalPayloadMissingRepositoryPrivateState.json: -------------------------------------------------------------------------------- 1 | { 2 | "commits": [ 3 | { } 4 | ], 5 | "head_commit": { 6 | "url": "https://github.com/octokitty/testing/commit/1481a2de7b2a7d02428ad93446ab166be7793fbb" 7 | }, 8 | "pusher": { 9 | "email": "lolwut@noway.biz", 10 | "name": "Garen Torikian" 11 | }, 12 | "ref": "refs/heads/master", 13 | "repository": { 14 | "name": "testing", 15 | "master_branch": "master", 16 | "language": "Ruby", 17 | "url": "https://github.com/octokitty/testing" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /subprojects/groovydsl/src/main/groovy/cd/pipeline/dsl/internal/DefaultAnnounceDsl.groovy: -------------------------------------------------------------------------------- 1 | package cd.pipeline.dsl.internal 2 | 3 | import cd.pipeline.dsl.AnnounceDsl 4 | import cd.pipeline.dsl.EmailDsl 5 | import cd.pipeline.messenger.MessageContext 6 | import cd.pipeline.util.ConfigureUtil 7 | 8 | class DefaultAnnounceDsl implements AnnounceDsl { 9 | final EmailDsl email = new DefaultEmailDsl() 10 | 11 | @Override 12 | void email(Closure config) { 13 | ConfigureUtil.configure(email, config) 14 | } 15 | 16 | List toContexts() { 17 | return [email].collect { it.toContext() } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /subprojects/groovydsl/src/test/groovy/cd/pipeline/dsl/PipelineDslSpec.groovy: -------------------------------------------------------------------------------- 1 | package cd.pipeline.dsl 2 | 3 | import cd.pipeline.api.Pipeline 4 | import cd.pipeline.dsl.internal.DefaultPipelineDsl 5 | import spock.lang.Specification 6 | 7 | class PipelineDslSpec extends Specification { 8 | 9 | def "Pipeline DSL exports to Pipeline implementation"() { 10 | PipelineDsl pipe = new DefaultPipelineDsl() 11 | 12 | when: 13 | pipe.stage "stageName", {} 14 | 15 | then: 16 | Pipeline actual = pipe.export() 17 | assert actual.stages != null 18 | assert actual.stages.size() == 1 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /subprojects/listener/src/main/groovy/cd/pipeline/listen/core/git/GithubUtils.groovy: -------------------------------------------------------------------------------- 1 | package cd.pipeline.listen.core.git 2 | 3 | class GithubUtils { 4 | 5 | private static final String SSH_URL_FORMAT = 'git@github.com:%s.git' 6 | 7 | static String toGithubSshUrl(String uri) throws URISyntaxException { 8 | return toGithubSshUrl(new URI(uri)) 9 | } 10 | 11 | static String toGithubSshUrl(URI uri) { 12 | return String.format(SSH_URL_FORMAT, cleanUriPath(uri)) 13 | } 14 | 15 | private static String cleanUriPath(URI uri) { 16 | def path = uri.path 17 | return path.replaceAll(/\/+/, '/') 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /subprojects/listener/src/test/groovy/cd/pipeline/listen/MainSpec.groovy: -------------------------------------------------------------------------------- 1 | package cd.pipeline.listen 2 | 3 | import spock.lang.Ignore 4 | import spock.lang.Specification 5 | 6 | class MainSpec extends Specification { 7 | 8 | @Ignore('Need to figure out how to mock the Main constructor or somehow better implemented Main') 9 | def 'Main should start the PipeListenService'() { 10 | def service = Mock(PipeListenService) 11 | def main = GroovySpy(Main, global: true) 12 | 13 | when: 14 | Main.main() 15 | 16 | then: 17 | 1 * new Main() 18 | 1 * main.createService() >> service 19 | 1 * service.run() 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /subprojects/groovydsl/src/main/groovy/cd/pipeline/dsl/EmailMessengerDsl.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.dsl; 2 | 3 | import cd.pipeline.messenger.internal.EmailMessenger; 4 | 5 | public interface EmailMessengerDsl { 6 | String getHost(); 7 | 8 | void setHost(String host); 9 | 10 | int getPort(); 11 | 12 | void setPort(int port); 13 | 14 | boolean getTls(); 15 | 16 | void setTls(boolean tls); 17 | 18 | String getUsername(); 19 | 20 | void setUsername(String username); 21 | 22 | String getPassword(); 23 | 24 | void setPassword(String password); 25 | 26 | String getFrom(); 27 | 28 | void setFrom(String from); 29 | 30 | EmailMessenger toMessenger(); 31 | } 32 | -------------------------------------------------------------------------------- /subprojects/runner/src/main/groovy/cd/pipeline/runner/cli/command/Command.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.runner.cli.command; 2 | 3 | public interface Command { 4 | 5 | /** 6 | * Get the name of the command. 7 | * 8 | * @return Name of the command, may never be null 9 | */ 10 | String getName(); 11 | 12 | /** 13 | * Can this command handle the given command name? 14 | * 15 | * @param commandName Command name asked to be handle 16 | * @return true if this command can handle the command name, false otherwise 17 | */ 18 | boolean canHandle(String commandName); 19 | 20 | /** 21 | * Handle this command. 22 | */ 23 | void handle(); 24 | } 25 | -------------------------------------------------------------------------------- /subprojects/groovydsl/src/main/groovy/cd/pipeline/messenger/internal/EmailContext.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.messenger.internal; 2 | 3 | import cd.pipeline.messenger.MessageContext; 4 | 5 | import java.util.Collections; 6 | import java.util.List; 7 | 8 | public class EmailContext implements MessageContext { 9 | private final List to; 10 | private final List cc; 11 | 12 | public EmailContext(List to, List cc) { 13 | this.to = Collections.unmodifiableList(to); 14 | this.cc = Collections.unmodifiableList(cc); 15 | } 16 | 17 | List getTo() { 18 | return to; 19 | } 20 | 21 | List getCc() { 22 | return cc; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /subprojects/runner/src/main/groovy/cd/pipeline/runner/internal/ServiceRegistry.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.runner.internal; 2 | 3 | import cd.pipeline.util.ServiceLookupException; 4 | 5 | public interface ServiceRegistry { 6 | 7 | /** 8 | * Locates the service of the given type. 9 | * 10 | * @param serviceType The service type. 11 | * @param The service type. 12 | * @return The service instance. Never returns null. 13 | * @throws UnknownServiceException When there is no service of the given type available. 14 | * @throws cd.pipeline.util.ServiceLookupException On failure to lookup the specified service. 15 | */ 16 | T get(Class serviceType) throws ServiceLookupException; 17 | } 18 | -------------------------------------------------------------------------------- /subprojects/groovydsl/src/main/groovy/cd/pipeline/dsl/internal/DefaultMessengerDsl.groovy: -------------------------------------------------------------------------------- 1 | package cd.pipeline.dsl.internal 2 | 3 | import cd.pipeline.dsl.EmailMessengerDsl 4 | import cd.pipeline.dsl.MessengerDsl 5 | import cd.pipeline.messenger.Messenger 6 | import cd.pipeline.messenger.internal.AggregateMessenger 7 | import cd.pipeline.util.ConfigureUtil 8 | 9 | class DefaultMessengerDsl implements MessengerDsl { 10 | final EmailMessengerDsl email = new DefaultEmailMessengerDsl() 11 | 12 | @Override 13 | void email(Closure config) { 14 | ConfigureUtil.configure(email, config) 15 | } 16 | 17 | @Override 18 | Messenger toMessenger() { 19 | return new AggregateMessenger([email].collect { it.toMessenger() }); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /docs/Technical-Design-Command-Ideas.md: -------------------------------------------------------------------------------- 1 | ## Project commands 2 | ### Setup project config 3 | * beam init 4 | 5 | This creates a .beam.config file with a template config. 6 | This config contains all properties of this project required to run the job on the daemon side. 7 | A sample config file looks like: 8 | TBA 9 | 10 | ## Daemon commands 11 | ### Setup the daemon workspace 12 | (which contains all configuration) 13 | * beam daemon init 14 | * because daemon init 15 | * bacon daemon init 16 | * bakeon daemon init 17 | 18 | ### Trigger a 'commit-job' based on given 'jobconfig.conf' 19 | From within the daemon workspace: 20 | * beam jobconfig.conf to commit-job 21 | * because commit-job runs jobconfig.conf 22 | * bacon jobconfig.conf with commit-job 23 | * bakeon jobconfig.conf with commit-job 24 | 25 | -------------------------------------------------------------------------------- /subprojects/listener/src/main/groovy/cd/pipeline/listen/core/healthcheck/GitCommandHealthCheck.groovy: -------------------------------------------------------------------------------- 1 | package cd.pipeline.listen.core.healthcheck 2 | 3 | import com.yammer.metrics.core.HealthCheck 4 | 5 | import static com.yammer.metrics.core.HealthCheck.Result.healthy 6 | import static com.yammer.metrics.core.HealthCheck.Result.unhealthy 7 | 8 | class GitCommandHealthCheck extends HealthCheck { 9 | 10 | GitCommandHealthCheck() { 11 | super('git command available') 12 | } 13 | 14 | @Override 15 | protected HealthCheck.Result check() throws Exception { 16 | def proc = "git --version".execute() 17 | if (proc.waitFor() == 0) { 18 | return healthy() 19 | } 20 | return unhealthy('[git] command not available on PATH') 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /gradle/requireJavaVersion7.gradle: -------------------------------------------------------------------------------- 1 | import static org.gradle.api.JavaVersion.VERSION_1_7 2 | 3 | if (!JavaVersion.current().isJava7Compatible()) { 4 | throw new RequireJava7CompatibleJavaVersion() 5 | } 6 | 7 | subprojects { 8 | if (project.plugins.hasPlugin('java')) { 9 | sourceCompatibility = VERSION_1_7 10 | targetCompatibility = VERSION_1_7 11 | 12 | test { 13 | testLogging { 14 | exceptionFormat = 'full' 15 | } 16 | } 17 | } 18 | } 19 | 20 | class RequireJava7CompatibleJavaVersion extends RuntimeException { 21 | @Override 22 | String getMessage() { 23 | final msg = "This project requires Java 7 or higher, you're running version '%s'" 24 | return String.format(msg, JavaVersion.current()) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /subprojects/api/src/main/groovy/cd/pipeline/event/PipeEvent.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.event; 2 | 3 | public class PipeEvent { 4 | private final String name; 5 | private final String status; 6 | private final String description; 7 | private final String details; 8 | 9 | public PipeEvent(String name, String status, String description, String details) { 10 | this.name = name; 11 | this.status = status; 12 | this.description = description; 13 | this.details = details; 14 | } 15 | 16 | public String getName() { 17 | return name; 18 | } 19 | 20 | public String getStatus() { 21 | return status; 22 | } 23 | 24 | public String getDescription() { 25 | return description; 26 | } 27 | 28 | public String getDetails() { 29 | return details; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /subprojects/groovydsl/src/main/groovy/cd/pipeline/messenger/internal/AggregateMessenger.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.messenger.internal; 2 | 3 | import cd.pipeline.event.PipeEvent; 4 | import cd.pipeline.messenger.MessageContext; 5 | import cd.pipeline.messenger.Messenger; 6 | import cd.pipeline.messenger.SpecializedMessenger; 7 | 8 | public class AggregateMessenger implements Messenger { 9 | private final Iterable messengers; 10 | 11 | public AggregateMessenger(Iterable messengers) { 12 | this.messengers = messengers; 13 | } 14 | 15 | public void process(MessageContext context, PipeEvent event) { 16 | for (SpecializedMessenger messenger : messengers) { 17 | if (messenger.accepts(context)) { 18 | messenger.process(context, event); 19 | } 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /subprojects/listener/src/main/groovy/cd/pipeline/listen/core/DeadEventHandler.groovy: -------------------------------------------------------------------------------- 1 | package cd.pipeline.listen.core 2 | 3 | import com.google.common.eventbus.DeadEvent 4 | import com.google.common.eventbus.EventBus 5 | import com.google.common.eventbus.Subscribe 6 | import com.yammer.dropwizard.lifecycle.Managed 7 | import groovy.util.logging.Slf4j 8 | 9 | @Slf4j 10 | class DeadEventHandler implements Managed { 11 | 12 | private final EventBus bus 13 | 14 | DeadEventHandler(EventBus bus) { 15 | this.bus = bus 16 | } 17 | 18 | @Override 19 | void start() throws Exception { 20 | bus.register(this) 21 | } 22 | 23 | @Override 24 | void stop() throws Exception { 25 | bus.unregister(this) 26 | } 27 | 28 | @Subscribe 29 | void work(DeadEvent event) { 30 | log.error("No event handler available for event [{}]", event.event) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /subprojects/listener/README.md: -------------------------------------------------------------------------------- 1 | Pipeline Listener 2 | ================= 3 | 4 | Receives and queues service hook notifications, like `push` events from GitHub. 5 | 6 | This listener supports the following provider: 7 | 8 | - github.com 9 | - gitlab.org/com 10 | 11 | Usage 12 | ----- 13 | 14 | Pipeline Listener (`pipe-listener`) provides a HTTP REST API that providers like GitHub can notify. The API is non-blocking, after basic verification of the notification message, the API will return a success (HTTP 204) or failure (HTTP 422) status. The notification message will be queued for processing a.s.a.p. by its processor. 15 | 16 | ### POST /providers/github 17 | 18 | GitHub provider, supports the GitHub WebHook format as explained on [Post-Receive Hooks](https://help.github.com/articles/post-receive-hooks) 19 | 20 | ### POST /providers/gitlab 21 | 22 | GitLab provider, supports the GitLab WebHook format as explained on [Web hooks] help page of your GitLab installation -------------------------------------------------------------------------------- /subprojects/listener/src/main/groovy/cd/pipeline/listen/core/healthcheck/PipeRunnerCommandHealthCheck.groovy: -------------------------------------------------------------------------------- 1 | package cd.pipeline.listen.core.healthcheck 2 | 3 | import com.yammer.metrics.core.HealthCheck 4 | 5 | import static com.yammer.metrics.core.HealthCheck.Result.healthy 6 | import static com.yammer.metrics.core.HealthCheck.Result.unhealthy 7 | 8 | class PipeRunnerCommandHealthCheck extends HealthCheck { 9 | 10 | PipeRunnerCommandHealthCheck() { 11 | super('pipe-runner command available') 12 | } 13 | 14 | @Override 15 | protected HealthCheck.Result check() throws Exception { 16 | def proc 17 | try { 18 | proc = "pipe-runner help".execute() 19 | } catch (IOException e) { 20 | return unhealthy(e.message) 21 | } 22 | if (proc.waitFor() == 0) { 23 | return healthy() 24 | } 25 | return unhealthy('[pipe-runner] command not available on PATH') 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /gradle/cucumber.gradle: -------------------------------------------------------------------------------- 1 | configurations { 2 | cucumberRuntime { 3 | extendsFrom testRuntime 4 | } 5 | } 6 | 7 | dependencies { 8 | testCompile group: 'info.cukes', name: 'cucumber-groovy', version: '1.1.2' 9 | testCompile group: 'info.cukes', name: 'cucumber-junit', version: '1.1.2', { 10 | exclude group: 'junit', module: 'junit' 11 | } 12 | 13 | cucumberRuntime files("${jar.archivePath}") 14 | } 15 | 16 | task cucumber { 17 | dependsOn assemble, test 18 | group = 'verification' 19 | description = 'Run the features.' 20 | doLast { 21 | javaexec { 22 | main = "cucumber.api.cli.Main" 23 | classpath = configurations.cucumberRuntime 24 | args = [ 25 | '-f' 26 | , 'pretty' 27 | , '--glue' 28 | , 'src/test/groovy/stepdefs' 29 | , 'src/test/resources/features' 30 | ] 31 | } 32 | } 33 | } 34 | 35 | check.dependsOn cucumber 36 | -------------------------------------------------------------------------------- /subprojects/runner/src/test/groovy/cd/pipeline/runner/internal/DefaultServiceRegistrySpec.groovy: -------------------------------------------------------------------------------- 1 | package cd.pipeline.runner.internal 2 | 3 | import cd.pipeline.dsl.PipelineDsl 4 | import cd.pipeline.dsl.internal.DefaultPipelineDsl 5 | import cd.pipeline.util.ServiceLookupException 6 | import spock.lang.Specification 7 | import spock.lang.Unroll 8 | 9 | class DefaultServiceRegistrySpec extends Specification { 10 | final registry = new DefaultServiceRegistry() 11 | 12 | @Unroll 13 | def "Provides #impl.simpleName for #api.simpleName"() { 14 | expect: 15 | assert impl.isInstance(registry.get(api)) 16 | 17 | where: 18 | api || impl 19 | PipelineDsl || DefaultPipelineDsl 20 | } 21 | 22 | def "Throws a ServiceLookupException when asking for an unknown implementation"() { 23 | when: 24 | registry.get(UnknownApi) 25 | 26 | then: 27 | def e = thrown(ServiceLookupException) 28 | assert e.message.contains(UnknownApi.class.simpleName) 29 | } 30 | } 31 | 32 | interface UnknownApi {} 33 | -------------------------------------------------------------------------------- /subprojects/groovydsl/src/main/groovy/cd/pipeline/dsl/internal/DefaultEmailMessengerDsl.groovy: -------------------------------------------------------------------------------- 1 | package cd.pipeline.dsl.internal 2 | 3 | import cd.pipeline.dsl.EmailMessengerDsl 4 | import cd.pipeline.messenger.internal.EmailMessenger 5 | import org.springframework.mail.javamail.JavaMailSender 6 | import org.springframework.mail.javamail.JavaMailSenderImpl 7 | 8 | class DefaultEmailMessengerDsl implements EmailMessengerDsl { 9 | String host 10 | int port 11 | boolean tls = false 12 | String username 13 | String password 14 | String from 15 | 16 | EmailMessenger toMessenger() { 17 | JavaMailSender sender = new JavaMailSenderImpl() 18 | sender.with { 19 | host = this.host 20 | port = this.port 21 | username = this.username 22 | password = this.password 23 | if (tls) { 24 | javaMailProperties = new Properties() 25 | javaMailProperties['mail.smtp.starttls.enable'] = 'true' 26 | } 27 | } 28 | return new EmailMessenger(sender, from) 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /docs/Home.md: -------------------------------------------------------------------------------- 1 | # Pipeline 2 | A Continuous Delivery (CD) tool, which we and all stakeholders of the CD pipeline love to use. Licensed under the MIT License 3 | 4 | > A free, open-source 5 | > Continuous Delivery tool 6 | > that's powerful enough for the enterprise, 7 | > flexible enough for startups 8 | > and simple to use for hobbyists 9 | 10 | ## Vision 11 | - [Why a new tool for Continuous Delivery](/pipelinelabs/pipeline/wiki/Vision-Why-a-new-tool-for-Continuous-Delivery) 12 | 13 | ## User Guide 14 | - [Using Pipeline with a Central Repository](/pipelinelabs/pipeline/wiki/User-Guide-Set-up-with-Central-Repository) 15 | - [Configuring the Pipeline Configuration for your project](/pipelinelabs/pipeline/wiki/User-Guide-Project-Pipeline-Configuration) 16 | 17 | ## Technical Design 18 | - [Braindump](/pipelinelabs/pipeline/wiki/Technical-Design-Braindump) 19 | - [System Design](/pipelinelabs/pipeline/wiki/Technical-Design-System-design) 20 | - [Command Ideas](/pipelinelabs/pipeline/wiki/Technical-Design-Command-Ideas) 21 | - [DSL Design Ideas](/pipelinelabs/pipeline/wiki/Technical-Design-DSL-Design-Ideas) -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2012-2013 Patrick van Dissel 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /subprojects/listener/src/test/groovy/cd/pipeline/listen/core/GitWorkerSpec.groovy: -------------------------------------------------------------------------------- 1 | package cd.pipeline.listen.core 2 | 3 | import com.google.common.eventbus.EventBus 4 | import com.google.common.eventbus.Subscribe 5 | import spock.lang.Ignore 6 | import spock.lang.Specification 7 | 8 | class GitWorkerSpec extends Specification { 9 | private EventBus bus = Mock(EventBus) 10 | private GitWorker worker = new GitWorker(bus) 11 | 12 | def 'Start registers itself to the event bus'() { 13 | when: 14 | worker.start() 15 | 16 | then: 17 | 1 * bus.register(worker) 18 | } 19 | 20 | def 'Stop unregisters itself from the event bus'() { 21 | when: 22 | worker.stop() 23 | 24 | then: 25 | 1 * bus.unregister(worker) 26 | } 27 | 28 | def 'work() is the event handler of GitTriggerEvent objects'() { 29 | def method = GitWorker.getMethod("work", GitTriggerEvent) 30 | 31 | expect: 32 | method.isAnnotationPresent(Subscribe) 33 | } 34 | 35 | @Ignore 36 | def 'Work clones the git repo and starts pipe-runner'() { 37 | final event = Mock(GitTriggerEvent) 38 | 39 | expect: 40 | worker.work(event) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /subprojects/graphs/src/test/groovy/cd/pipeline/graphs/GraphSpec.groovy: -------------------------------------------------------------------------------- 1 | package cd.pipeline.graphs 2 | 3 | import spock.lang.Specification 4 | 5 | class GraphSpec extends Specification { 6 | /** 7 | * - two stages executed in order of adding 8 | * - three stages where 1,2 are executed at the same time, 3 is executed after 1 9 | * - four stages where 1,2 are executed at the same time, 3 is executed after 1, 4 is executed after 2 10 | * - four stages where 2 executes after 1, 3 and 4 are executed after 2 11 | */ 12 | 13 | def "Can execute stages after eachother"() { 14 | given: "multiple stages are added as dependend on eachother" 15 | 16 | when: "the pipeline is executed" 17 | 18 | then: "each next stage is executed when the previous one finishes successfully" 19 | } 20 | 21 | def "Can execute stages in parallel"() { 22 | } 23 | 24 | def "Multiple stages can join together to one next stage"() { 25 | given: "multiple stages join together to one next stage" 26 | 27 | when: "the next stage is reached" 28 | 29 | then: "the stage is not executed until all joining stages are ready" 30 | } 31 | 32 | def "Can execute stages in parallel and join"() { 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /subprojects/runner/src/main/groovy/cd/pipeline/runner/internal/DefaultServiceRegistry.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.runner.internal; 2 | 3 | import cd.pipeline.dsl.internal.DefaultPipelineDsl; 4 | import cd.pipeline.util.ServiceLookupException; 5 | 6 | public class DefaultServiceRegistry implements ServiceRegistry { 7 | 8 | @Override 9 | @SuppressWarnings("unchecked") 10 | public T get(final Class serviceType) throws ServiceLookupException { 11 | final T service; 12 | switch (serviceType.getSimpleName()) { 13 | case "PipelineDsl": 14 | service = (T) new DefaultPipelineDsl(); 15 | break; 16 | default: 17 | throw new UnsupportedServiceType(serviceType); 18 | } 19 | return service; 20 | } 21 | 22 | private class UnsupportedServiceType extends ServiceLookupException { 23 | 24 | private Class clazz; 25 | 26 | UnsupportedServiceType(final Class clazz) { 27 | this.clazz = clazz; 28 | } 29 | 30 | @Override 31 | public String getMessage() { 32 | final String msg = "Service type '%s' is not supported"; 33 | return String.format(msg, clazz.getSimpleName()); 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /subprojects/listener/src/test/groovy/cd/pipeline/listen/core/DeadEventHandlerSpec.groovy: -------------------------------------------------------------------------------- 1 | package cd.pipeline.listen.core 2 | 3 | import com.google.common.eventbus.DeadEvent 4 | import com.google.common.eventbus.EventBus 5 | import com.google.common.eventbus.Subscribe 6 | import spock.lang.Ignore 7 | import spock.lang.Specification 8 | 9 | class DeadEventHandlerSpec extends Specification { 10 | private EventBus bus = Mock(EventBus) 11 | private DeadEventHandler worker = new DeadEventHandler(bus) 12 | 13 | def 'Start registers itself to the event bus'() { 14 | when: 15 | worker.start() 16 | 17 | then: 18 | 1 * bus.register(worker) 19 | } 20 | 21 | def 'Stop unregisters itself from the event bus'() { 22 | when: 23 | worker.stop() 24 | 25 | then: 26 | 1 * bus.unregister(worker) 27 | } 28 | 29 | def 'work() is the event handler of DeadEvent objects'() { 30 | def method = DeadEventHandler.getMethod("work", DeadEvent) 31 | 32 | expect: 33 | method.isAnnotationPresent(Subscribe) 34 | } 35 | 36 | @Ignore('How to verify if injected logger is used?') 37 | def 'Work logs an error'() { 38 | def worker = new DeadEventHandler(bus) 39 | final event = Mock(DeadEvent) 40 | 41 | expect: 42 | worker.work(event) 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /subprojects/runner/src/test/groovy/stepdefs/CliStep.groovy: -------------------------------------------------------------------------------- 1 | package stepdefs 2 | 3 | import cd.pipeline.runner.cli.Main 4 | import cucumber.api.groovy.EN 5 | import cucumber.api.groovy.Hooks 6 | 7 | this.metaClass.mixin Hooks 8 | this.metaClass.mixin EN 9 | 10 | def Main cli 11 | def output 12 | 13 | Given(~'^the command line application$') { -> 14 | cli = new Main() 15 | } 16 | 17 | When(~'^I provide (.*) as parameter$') { param -> 18 | if (param == '') { 19 | param = [] as String 20 | } 21 | def orgOut = System.out 22 | System.out = output 23 | final exitCode = cli.run('pipe-runner', param) 24 | System.out = orgOut 25 | (exitCode == 0) 26 | } 27 | 28 | Then(~'^I expect to see the application help$') { -> 29 | def help = ''' 30 | Usage: pipe-runner [options] [command] [command options] 31 | Options: 32 | --help, -h 33 | 34 | Default: false 35 | Commands: 36 | help Show help information about a command 37 | Usage: help [options] 38 | 39 | run Run, baby run! 40 | Usage: run [options] 41 | 42 | feedback Give feedback about your experience with this tool 43 | Usage: feedback [options] 44 | Options: 45 | -n, --negative 46 | Your negative feedback message 47 | -p, --positive 48 | Your positive feedback message 49 | ''' 50 | 51 | (help == output) 52 | } 53 | -------------------------------------------------------------------------------- /subprojects/runner/src/main/groovy/cd/pipeline/runner/cli/command/HelpCommand.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.runner.cli.command; 2 | 3 | import com.beust.jcommander.JCommander; 4 | import com.beust.jcommander.Parameters; 5 | 6 | import java.io.PrintStream; 7 | 8 | @Parameters( 9 | commandNames = {HelpCommand.NAME} 10 | , commandDescription = "Show help information about a command" 11 | ) 12 | public class HelpCommand implements Command { 13 | public static final String NAME = "help"; 14 | 15 | private final PrintStream outputConsumer; 16 | private final JCommander cli; 17 | 18 | public HelpCommand(final PrintStream outputConsumer, final JCommander cli) { 19 | this.outputConsumer = outputConsumer; 20 | this.cli = cli; 21 | } 22 | 23 | @Override 24 | public String getName() { 25 | return NAME; 26 | } 27 | 28 | @Override 29 | public boolean canHandle(final String commandName) { 30 | return getName().equals(commandName); 31 | } 32 | 33 | @Override 34 | public void handle() { 35 | outputUsage(); 36 | } 37 | 38 | public void outputUsage() { 39 | outputUsage(outputConsumer); 40 | } 41 | 42 | private void outputUsage(final PrintStream output) { 43 | final StringBuilder stringOutput = new StringBuilder(); 44 | cli.usage(stringOutput); 45 | output.print(stringOutput); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /subprojects/listener/src/test/groovy/cd/pipeline/listen/core/git/GithubUtilsSpec.groovy: -------------------------------------------------------------------------------- 1 | package cd.pipeline.listen.core.git 2 | 3 | import spock.lang.Specification 4 | import spock.lang.Unroll 5 | 6 | class GithubUtilsSpec extends Specification { 7 | 8 | @Unroll 9 | def 'Convert new URI("#uri") to github ssh url "#expected"'() { 10 | expect: 11 | final actual = GithubUtils.toGithubSshUrl(new URI(uri)) 12 | actual == expected 13 | 14 | where: 15 | uri || expected 16 | 'https://github.com/octokitty/testing' || 'git@github.com:/octokitty/testing.git' 17 | 'http://github.com/octokitty/testing' || 'git@github.com:/octokitty/testing.git' 18 | 'http://github.com//octokitty/testing' || 'git@github.com:/octokitty/testing.git' 19 | 'http://github.com/octokitty//testing' || 'git@github.com:/octokitty/testing.git' 20 | } 21 | 22 | @Unroll 23 | def 'Convert "#uri" to github ssh url "#expected"'() { 24 | expect: 25 | final actual = GithubUtils.toGithubSshUrl(uri) 26 | actual == expected 27 | 28 | where: 29 | uri || expected 30 | 'https://github.com/octokitty/testing' || 'git@github.com:/octokitty/testing.git' 31 | 'http://github.com/octokitty/testing' || 'git@github.com:/octokitty/testing.git' 32 | 'http://github.com//octokitty/testing' || 'git@github.com:/octokitty/testing.git' 33 | 'http://github.com/octokitty//testing' || 'git@github.com:/octokitty/testing.git' 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /subprojects/listener/src/test/resources/gitlab/samplePayload.json: -------------------------------------------------------------------------------- 1 | { 2 | "after": "caef9001d8a6e86e798a15d5ce4b87629e959a06", 3 | "before": "a1a0d677c3414dce9d0b64a6573161e295ec3842", 4 | "commits": [ 5 | { 6 | "author": { 7 | "email": "lolwut@noway.biz", 8 | "name": "Garen Torikian" 9 | }, 10 | "id": "caef9001d8a6e86e798a15d5ce4b87629e959a06", 11 | "message": "Test", 12 | "timestamp": "2013-10-04T11:33:01+02:00", 13 | "url": "https://git.domain.com/group/project/commit/caef9001d8a6e86e798a15d5ce4b87629e959a06" 14 | }, 15 | { 16 | "author": { 17 | "email": "lolwut@noway.biz", 18 | "name": "Garen Torikian" 19 | }, 20 | "id": "93bed336334536f375041bbb84ed20415f902f30", 21 | "message": "Test 2", 22 | "timestamp": "2013-10-04T11:26:42+02:00", 23 | "url": "https://git.domain.com/group/project/commit/93bed336334536f375041bbb84ed20415f902f30" 24 | }, 25 | { 26 | "author": { 27 | "email": "lolwut@noway.biz", 28 | "name": "Garen Torikian" 29 | }, 30 | "id": "a1a0d677c3414dce9d0b64a6573161e295ec3842", 31 | "message": "Test 3", 32 | "timestamp": "2013-10-04T11:08:45+02:00", 33 | "url": "https://git.domain.com/group/project/commit/a1a0d677c3414dce9d0b64a6573161e295ec3842" 34 | } 35 | ], 36 | "ref": "refs/heads/master", 37 | "repository": { 38 | "description": "", 39 | "homepage": "https://git.domain.com/group/project", 40 | "name": "project", 41 | "url": "git@git.domain.com:group/project.git" 42 | }, 43 | "total_commits_count": 3, 44 | "user_id": 5, 45 | "user_name": "Garen Torikian" 46 | } -------------------------------------------------------------------------------- /subprojects/listener/src/test/resources/gitlab/samplePayloadFromBranch.json: -------------------------------------------------------------------------------- 1 | { 2 | "after": "caef9001d8a6e86e798a15d5ce4b87629e959a06", 3 | "before": "a1a0d677c3414dce9d0b64a6573161e295ec3842", 4 | "commits": [ 5 | { 6 | "author": { 7 | "email": "lolwut@noway.biz", 8 | "name": "Garen Torikian" 9 | }, 10 | "id": "caef9001d8a6e86e798a15d5ce4b87629e959a06", 11 | "message": "Test", 12 | "timestamp": "2013-10-04T11:33:01+02:00", 13 | "url": "https://git.domain.com/group/project/commit/caef9001d8a6e86e798a15d5ce4b87629e959a06" 14 | }, 15 | { 16 | "author": { 17 | "email": "lolwut@noway.biz", 18 | "name": "Garen Torikian" 19 | }, 20 | "id": "93bed336334536f375041bbb84ed20415f902f30", 21 | "message": "Test 2", 22 | "timestamp": "2013-10-04T11:26:42+02:00", 23 | "url": "https://git.domain.com/group/project/commit/93bed336334536f375041bbb84ed20415f902f30" 24 | }, 25 | { 26 | "author": { 27 | "email": "lolwut@noway.biz", 28 | "name": "Garen Torikian" 29 | }, 30 | "id": "a1a0d677c3414dce9d0b64a6573161e295ec3842", 31 | "message": "Test 3", 32 | "timestamp": "2013-10-04T11:08:45+02:00", 33 | "url": "https://git.domain.com/group/project/commit/a1a0d677c3414dce9d0b64a6573161e295ec3842" 34 | } 35 | ], 36 | "ref": "refs/heads/somebranch", 37 | "repository": { 38 | "description": "", 39 | "homepage": "https://git.domain.com/group/project", 40 | "name": "project", 41 | "url": "git@git.domain.com:group/project.git" 42 | }, 43 | "total_commits_count": 3, 44 | "user_id": 5, 45 | "user_name": "Garen Torikian" 46 | } -------------------------------------------------------------------------------- /subprojects/listener/src/main/groovy/cd/pipeline/listen/PipeListenService.groovy: -------------------------------------------------------------------------------- 1 | package cd.pipeline.listen 2 | 3 | import cd.pipeline.listen.core.DeadEventHandler 4 | import cd.pipeline.listen.core.GitWorker 5 | import cd.pipeline.listen.core.healthcheck.GitCommandHealthCheck 6 | import cd.pipeline.listen.core.healthcheck.PipeRunnerCommandHealthCheck 7 | import cd.pipeline.listen.resources.GitHubWebHookResource 8 | import cd.pipeline.listen.resources.GitLabWebHookResource 9 | import com.google.common.eventbus.AsyncEventBus 10 | import com.yammer.dropwizard.Service 11 | import com.yammer.dropwizard.config.Bootstrap 12 | import com.yammer.dropwizard.config.Environment 13 | 14 | import static java.util.concurrent.TimeUnit.SECONDS 15 | 16 | class PipeListenService extends Service { 17 | 18 | @Override 19 | void initialize(Bootstrap bootstrap) { 20 | } 21 | 22 | @Override 23 | void run(PipeListenConfiguration config, Environment env) throws Exception { 24 | def bus = createEventBus(env) 25 | env.manage(new DeadEventHandler(bus)) 26 | env.manage(new GitWorker(bus)); 27 | env.addResource(new GitHubWebHookResource(bus)) 28 | env.addResource(new GitLabWebHookResource(bus)) 29 | env.addHealthCheck(new GitCommandHealthCheck()) 30 | env.addHealthCheck(new PipeRunnerCommandHealthCheck()) 31 | } 32 | 33 | private createEventBus(Environment env) { 34 | final corePoolSize = 2 35 | final maxPoolSize = 6 36 | final keepAliveTimeInSeconds = 30 37 | new AsyncEventBus( 38 | env.managedExecutorService( 39 | "eventbus-worker-%s", 40 | corePoolSize, 41 | maxPoolSize, 42 | keepAliveTimeInSeconds, SECONDS 43 | ) 44 | ) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /subprojects/listener/src/main/groovy/cd/pipeline/listen/core/GitWorker.groovy: -------------------------------------------------------------------------------- 1 | package cd.pipeline.listen.core 2 | 3 | import com.google.common.eventbus.EventBus 4 | import com.google.common.eventbus.Subscribe 5 | import com.yammer.dropwizard.lifecycle.Managed 6 | import groovy.util.logging.Slf4j 7 | 8 | import java.nio.file.Files 9 | import java.nio.file.Path 10 | import java.nio.file.Paths 11 | 12 | @Slf4j 13 | class GitWorker implements Managed { 14 | 15 | private final EventBus bus 16 | 17 | GitWorker(EventBus bus) { 18 | this.bus = bus 19 | } 20 | 21 | @Override 22 | void start() throws Exception { 23 | bus.register(this) 24 | } 25 | 26 | @Override 27 | void stop() throws Exception { 28 | bus.unregister(this) 29 | } 30 | 31 | @Subscribe 32 | void work(GitTriggerEvent event) { 33 | log.info('Received event [{}]', event) 34 | def dir = Files.createTempDirectory("pipeline") 35 | log.debug('Created pipeline directory [{}]', dir) 36 | 37 | def workDirName = "work" 38 | def workDir = Paths.get(dir.toString(), workDirName) 39 | List envProps = null 40 | 41 | def gitCloneCommand = "git clone ${event.url} ${workDirName}" 42 | executeCommand(gitCloneCommand, dir, envProps) 43 | 44 | def gitCheckoutCommand = "git checkout --force ${event.branch}" 45 | executeCommand(gitCheckoutCommand, workDir, envProps) 46 | 47 | def runnerCommand = "pipe-runner run project.pipe" 48 | executeCommand(runnerCommand, workDir, envProps) 49 | } 50 | 51 | private void executeCommand(String command, Path workDir, List envProps) { 52 | log.info('Executing command [{}] in pipeline directory [{}]', command, workDir) 53 | def runnerProc = command.execute(envProps, workDir.toFile()) 54 | def runnerExitStatus = runnerProc.waitFor() 55 | log.info('Command [{}] exited with status [{}]', command, runnerExitStatus) 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /subprojects/runner/src/main/groovy/cd/pipeline/runner/cli/command/FeedbackCommand.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.runner.cli.command; 2 | 3 | import com.beust.jcommander.Parameter; 4 | import com.beust.jcommander.Parameters; 5 | 6 | import java.io.PrintStream; 7 | 8 | @Parameters( 9 | commandNames = {FeedbackCommand.NAME} 10 | , commandDescription = "Give feedback about your experience with this tool" 11 | ) 12 | public class FeedbackCommand implements Command { 13 | public static final String NAME = "feedback"; 14 | 15 | @Parameter( 16 | names = { 17 | "-p" 18 | , "--positive" 19 | } 20 | , description = "Your positive feedback message" 21 | ) 22 | private String positive; 23 | 24 | @Parameter( 25 | names = { 26 | "-n" 27 | , "--negative" 28 | } 29 | , description = "Your negative feedback message" 30 | ) 31 | private String negative; 32 | 33 | private PrintStream outputConsumer; 34 | 35 | public FeedbackCommand(final PrintStream outputConsumer) { 36 | this.outputConsumer = outputConsumer; 37 | } 38 | 39 | @Override 40 | public String getName() { 41 | return NAME; 42 | } 43 | 44 | @Override 45 | public boolean canHandle(final String commandName) { 46 | return getName().equals(commandName); 47 | } 48 | 49 | @Override 50 | public void handle() { 51 | String feedbackType; 52 | String msg; 53 | if (negative != null && !negative.isEmpty()) { 54 | feedbackType = "negative"; 55 | msg = negative; 56 | } else if (positive != null && !positive.isEmpty()) { 57 | feedbackType = "positive"; 58 | msg = positive; 59 | } else { 60 | return; 61 | } 62 | outputConsumer.printf("Thnx, received your %s feedback: %s\n", feedbackType, msg); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /subprojects/runner/src/main/groovy/cd/pipeline/runner/cli/dsl/PipelineScriptRunner.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.runner.cli.dsl; 2 | 3 | import cd.pipeline.dsl.PipelineDsl; 4 | import cd.pipeline.runner.cli.dsl.script.PipelineScript; 5 | import cd.pipeline.runner.internal.ServiceRegistry; 6 | import groovy.lang.GroovyShell; 7 | import org.codehaus.groovy.control.CompilerConfiguration; 8 | 9 | import java.io.PrintStream; 10 | 11 | public class PipelineScriptRunner { 12 | 13 | private final PipelineScript script; 14 | private final PrintStream redirectedOutput; 15 | private PrintStream originalStdOut; 16 | private PrintStream originalStdErr; 17 | 18 | public PipelineScriptRunner(final ServiceRegistry registry, final PrintStream output, final String scriptText) { 19 | redirectedOutput = output; 20 | final CompilerConfiguration config = new CompilerConfiguration(); 21 | config.setScriptBaseClass(PipelineScript.class.getName()); 22 | 23 | final GroovyShell shell = new GroovyShell(config); 24 | script = (PipelineScript) shell.parse(scriptText); 25 | 26 | final PipelineDsl pipeline = registry.get(PipelineDsl.class); 27 | script.init(pipeline); 28 | } 29 | 30 | public void run() { 31 | try { 32 | startCapturingOutput(); 33 | script.run(); 34 | } finally { 35 | stopCapturingOutput(); 36 | } 37 | } 38 | 39 | private void startCapturingOutput() { 40 | originalStdOut = System.out; 41 | System.setOut(redirectedOutput); 42 | originalStdErr = System.err; 43 | System.setErr(redirectedOutput); 44 | } 45 | 46 | private void stopCapturingOutput() { 47 | try { 48 | System.setOut(originalStdOut); 49 | System.setErr(originalStdErr); 50 | redirectedOutput.flush(); 51 | } finally { 52 | originalStdOut = null; 53 | originalStdErr = null; 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /gradle/dependencies.gradle: -------------------------------------------------------------------------------- 1 | ext { 2 | libraries = [:] 3 | versions = [:] 4 | } 5 | 6 | versions += [ 7 | groovy: '2.1.7', 8 | dropwizard: '0.6.2', 9 | ] 10 | 11 | libraries += [ 12 | mavenSharedUtils: 'org.apache.maven.shared:maven-shared-utils:0.4', 13 | groovy: "org.codehaus.groovy:groovy:${versions.groovy}", 14 | groovyJson: "org.codehaus.groovy:groovy-json:${versions.groovy}", 15 | mail: 'javax.mail:mail:1.4.7', 16 | springContextSupport: 'org.springframework:spring-context-support:3.2.4.RELEASE', 17 | 18 | junit: 'junit:junit:4.11', 19 | asm: 'org.ow2.asm:asm:4.1', 20 | hamcrest: 'org.hamcrest:hamcrest-library:1.3', 21 | jcommander: 'com.beust:jcommander:1.32', 22 | dropwizardTesting: "com.yammer.dropwizard:dropwizard-testing:${versions.dropwizard}", 23 | ] 24 | 25 | libraries.dropwizard = ["com.yammer.dropwizard:dropwizard-core:${versions.dropwizard}", 26 | 'com.sun.jersey:jersey-client:1.17.1', 27 | ] 28 | 29 | libraries.spock = ['org.spockframework:spock-core:0.7-groovy-2.0', 30 | libraries.groovy, 31 | 'org.objenesis:objenesis:2.0', 32 | 'cglib:cglib-nodep:2.2' 33 | ] 34 | 35 | allprojects { 36 | configurations.all { 37 | resolutionStrategy.eachDependency { DependencyResolveDetails details -> 38 | def req = details.requested 39 | // Replace groovy-all with groovy 40 | if (req.name == 'groovy-all' || req.name == 'groovy') { 41 | details.useTarget libraries.groovy 42 | } 43 | // Replace junit-dep with junit 44 | if (req.name == 'junit-dep') { 45 | details.useTarget libraries.junit 46 | } 47 | // Change old groupId of asm to new groupId, for correct dependency handling 48 | if (req.group == 'asm') { 49 | details.useTarget group: 'org.ow2.asm', name: req.name, version: req.version 50 | } 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /subprojects/api/src/test/groovy/cd/pipeline/util/ServiceLocatorSpec.groovy: -------------------------------------------------------------------------------- 1 | package cd.pipeline.util 2 | 3 | import spock.lang.Specification 4 | 5 | class ServiceLocatorSpec extends Specification { 6 | final locator = new ServiceLocator() 7 | 8 | def "Can locate a single implementation instance"() { 9 | when: 10 | def clazz = locator.find(ServiceExpectedToBeImplementedByOneClass) 11 | 12 | then: 13 | assert clazz instanceof SingleImplementation 14 | } 15 | 16 | def "Can locate multiple implementation instances"() { 17 | when: 18 | def clazz = locator.findAll(ServiceExpectedToBeImplementedByMultipleClasses) 19 | 20 | then: 21 | assert clazz instanceof Set 22 | assert clazz.size() == 3 23 | } 24 | 25 | def "Throws a ServiceLookupException if no implementation can be found"() { 26 | when: 27 | locator.find(ServiceWithoutAvailableImplementation) 28 | 29 | then: 30 | def e = thrown(ServiceLookupException) 31 | assert e.message.contains(ServiceWithoutAvailableImplementation.class.simpleName) 32 | } 33 | 34 | def "Throws a ServiceLookupException when expecting one implementation while finding multiple"() { 35 | when: 36 | locator.find(ServiceExpectedToBeImplementedByMultipleClasses) 37 | 38 | then: 39 | def e = thrown(ServiceLookupException) 40 | assert e.message.contains(ServiceExpectedToBeImplementedByMultipleClasses.class.simpleName) 41 | } 42 | } 43 | 44 | interface ServiceExpectedToBeImplementedByOneClass {} 45 | 46 | class SingleImplementation implements ServiceExpectedToBeImplementedByOneClass {} 47 | 48 | interface ServiceExpectedToBeImplementedByMultipleClasses {} 49 | 50 | class MultipleImplementationsFirstImpl implements ServiceExpectedToBeImplementedByMultipleClasses {} 51 | 52 | class MultipleImplementationsSecondImpl implements ServiceExpectedToBeImplementedByMultipleClasses {} 53 | 54 | class MultipleImplementationsThirdImpl implements ServiceExpectedToBeImplementedByMultipleClasses {} 55 | 56 | interface ServiceWithoutAvailableImplementation {} 57 | -------------------------------------------------------------------------------- /subprojects/runner/src/main/groovy/cd/pipeline/runner/cli/command/RunCommand.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.runner.cli.command; 2 | 3 | import cd.pipeline.runner.cli.dsl.PipelineScriptRunner; 4 | import cd.pipeline.runner.internal.ServiceRegistry; 5 | import com.beust.jcommander.Parameter; 6 | import com.beust.jcommander.Parameters; 7 | import com.beust.jcommander.converters.FileConverter; 8 | 9 | import java.io.File; 10 | import java.io.IOException; 11 | import java.io.PrintStream; 12 | import java.nio.file.Files; 13 | import java.util.ArrayList; 14 | import java.util.List; 15 | 16 | @Parameters( 17 | commandNames = {RunCommand.NAME} 18 | , commandDescription = "Run pipeline" 19 | ) 20 | public class RunCommand implements Command { 21 | public static final String NAME = "run"; 22 | 23 | @Parameter( 24 | description = "Pipeline configuration to run" 25 | , listConverter = FileConverter.class 26 | , required = true 27 | , arity = 1 28 | ) 29 | private List files = new ArrayList(2); 30 | 31 | private final PrintStream outputConsumer; 32 | private final ServiceRegistry registry; 33 | 34 | public RunCommand(final PrintStream outputConsumer, ServiceRegistry registry) { 35 | this.outputConsumer = outputConsumer; 36 | this.registry = registry; 37 | } 38 | 39 | @Override 40 | public String getName() { 41 | return NAME; 42 | } 43 | 44 | @Override 45 | public boolean canHandle(final String commandName) { 46 | return getName().equals(commandName); 47 | } 48 | 49 | @Override 50 | public void handle() { 51 | final File file = files.get(0); 52 | final String script; 53 | try { 54 | script = new String(Files.readAllBytes(file.toPath())); 55 | } catch (IOException e) { 56 | final String msg = String.format("Could not read script [%s]", file.getAbsolutePath()); 57 | throw new IllegalArgumentException(msg, e); 58 | } 59 | final PipelineScriptRunner pipeline = new PipelineScriptRunner(registry, outputConsumer, script); 60 | pipeline.run(); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /subprojects/groovydsl/src/test/groovy/cd/pipeline/messenger/internal/AggregateMessengerSpec.groovy: -------------------------------------------------------------------------------- 1 | package cd.pipeline.messenger.internal 2 | 3 | import cd.pipeline.event.PipeEvent 4 | import cd.pipeline.messenger.MessageContext 5 | import cd.pipeline.messenger.SpecializedMessenger 6 | import spock.lang.Specification 7 | 8 | class AggregateMessengerSpec extends Specification { 9 | MessageContext context = Mock() 10 | PipeEvent event = new PipeEvent('', '', '', '') 11 | 12 | def 'if no messengers, does nothing'() { 13 | given: 14 | def aggregate = new AggregateMessenger([]) 15 | when: 16 | aggregate.process(context, event) 17 | then: 18 | notThrown(Exception) 19 | } 20 | 21 | def 'if no messengers accept context, does not process'() { 22 | given: 23 | def messenger1 = Mock(SpecializedMessenger) 24 | def messenger2 = Mock(SpecializedMessenger) 25 | def aggregate = new AggregateMessenger([messenger1, messenger2]) 26 | when: 27 | aggregate.process(context, event) 28 | then: 29 | 1 * messenger1.accepts(_) >> false 30 | 1 * messenger2.accepts(_) >> false 31 | 0 * messenger1.process(_, _) 32 | 0 * messenger2.process(_, _) 33 | } 34 | 35 | def 'only messengers that accept context process the event'() { 36 | given: 37 | def messenger1 = Mock(SpecializedMessenger) 38 | def messenger2 = Mock(SpecializedMessenger) 39 | def messenger3 = Mock(SpecializedMessenger) 40 | def messenger4 = Mock(SpecializedMessenger) 41 | def aggregate = new AggregateMessenger([messenger1, messenger2, 42 | messenger3, messenger4]) 43 | when: 44 | aggregate.process(context, event) 45 | then: 46 | 1 * messenger1.accepts(context) >> false 47 | 1 * messenger2.accepts(context) >> true 48 | 1 * messenger3.accepts(context) >> true 49 | 1 * messenger4.accepts(context) >> false 50 | 0 * messenger1.process(_, _) 51 | 1 * messenger2.process(context, event) 52 | 1 * messenger3.process(context, event) 53 | 0 * messenger4.process(_, _) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /subprojects/groovydsl/src/test/groovy/cd/pipeline/messenger/internal/EmailMessengerSpec.groovy: -------------------------------------------------------------------------------- 1 | package cd.pipeline.messenger.internal 2 | 3 | import cd.pipeline.event.PipeEvent 4 | import cd.pipeline.messenger.MessageContext 5 | import org.springframework.mail.javamail.JavaMailSender 6 | import spock.lang.Specification 7 | 8 | import javax.mail.Message.RecipientType 9 | import javax.mail.Session 10 | import javax.mail.internet.MimeMessage 11 | 12 | class EmailMessengerSpec extends Specification { 13 | 14 | def 'accepts EmailContexts'() { 15 | expect: 16 | new EmailMessenger(null, null).accepts(new EmailContext([], [])) 17 | } 18 | 19 | def 'does not accept other contexts'() { 20 | expect: 21 | !new EmailMessenger(null, null).accepts(Mock(MessageContext)) 22 | } 23 | 24 | def 'process generates the correct email'() { 25 | given: 26 | PipeEvent event = new PipeEvent( 27 | 'MyProject', 28 | 'Success', 29 | 'Gradle build succeeded in 10 seconds.', 30 | 'a bunch of\nlog data\n') 31 | def from = '1@2.com' 32 | def to = ['a@b.com', 'c@d.com'] 33 | def cc = ['e@f.com'] 34 | EmailContext context = new EmailContext(to, cc) 35 | 36 | def email = new MimeMessage(null as Session) 37 | def sender = Mock(JavaMailSender) 38 | def messenger = new EmailMessenger(sender, from) 39 | 40 | when: 41 | messenger.process(context, event) 42 | 43 | then: 44 | 1 * sender.createMimeMessage() >> { email } 45 | 1 * sender.send({ MimeMessage msg -> 46 | msg.from.collect { it as String } == [from] && 47 | msg.getRecipients(RecipientType.TO).collect { it as String } == to && 48 | msg.getRecipients(RecipientType.CC).collect { it as String } == cc && 49 | msg.subject == '[pipeline] MyProject: Success' && 50 | msg.content == """\ 51 | Pipeline Name: MyProject 52 | Status: Success 53 | Description: Gradle build succeeded in 10 seconds. 54 | Details: 55 | a bunch of 56 | log data 57 | """.stripIndent() 58 | } as MimeMessage) 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /subprojects/listener/src/test/groovy/cd/pipeline/listen/PipeListenServiceSpec.groovy: -------------------------------------------------------------------------------- 1 | package cd.pipeline.listen 2 | 3 | import cd.pipeline.listen.core.DeadEventHandler 4 | import cd.pipeline.listen.core.GitWorker 5 | import cd.pipeline.listen.resources.GitHubWebHookResource 6 | import cd.pipeline.listen.resources.GitLabWebHookResource 7 | import com.yammer.dropwizard.config.Environment 8 | import spock.lang.Specification 9 | 10 | import java.util.concurrent.ExecutorService 11 | 12 | class PipeListenServiceSpec extends Specification { 13 | 14 | def 'Service serves the GitHubWebHookResource'() { 15 | given: 16 | def env = Mock(Environment) 17 | def executor = Mock(ExecutorService) 18 | def service = new PipeListenService() 19 | def config = new PipeListenConfiguration() 20 | 21 | when: 22 | service.run(config, env) 23 | 24 | then: 25 | 1 * env.managedExecutorService(_, _, _, _, _) >> executor 26 | 1 * env.addResource(_ as GitHubWebHookResource) 27 | } 28 | 29 | def 'Service serves the GitLabWebHookResource'() { 30 | given: 31 | def env = Mock(Environment) 32 | def executor = Mock(ExecutorService) 33 | def service = new PipeListenService() 34 | def config = new PipeListenConfiguration() 35 | 36 | when: 37 | service.run(config, env) 38 | 39 | then: 40 | 1 * env.managedExecutorService(_, _, _, _, _) >> executor 41 | 1 * env.addResource(_ as GitLabWebHookResource) 42 | } 43 | 44 | def 'Service manages the GitWorker'() { 45 | given: 46 | def env = Mock(Environment) 47 | def executor = Mock(ExecutorService) 48 | def service = new PipeListenService() 49 | def config = new PipeListenConfiguration() 50 | 51 | when: 52 | service.run(config, env) 53 | 54 | then: 55 | 1 * env.managedExecutorService(_, _, _, _, _) >> executor 56 | 1 * env.manage(_ as GitWorker) 57 | } 58 | 59 | def 'Service manages the DeadEventHandler'() { 60 | given: 61 | def env = Mock(Environment) 62 | def executor = Mock(ExecutorService) 63 | def service = new PipeListenService() 64 | def config = new PipeListenConfiguration() 65 | 66 | when: 67 | service.run(config, env) 68 | 69 | then: 70 | 1 * env.managedExecutorService(_, _, _, _, _) >> executor 71 | 1 * env.manage(_ as DeadEventHandler) 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /subprojects/listener/src/main/groovy/cd/pipeline/listen/resources/GitLabWebHookResource.groovy: -------------------------------------------------------------------------------- 1 | package cd.pipeline.listen.resources 2 | 3 | import cd.pipeline.listen.core.GitTriggerEvent 4 | import com.google.common.eventbus.EventBus 5 | import groovy.json.JsonSlurper 6 | import groovy.util.logging.Slf4j 7 | 8 | import javax.ws.rs.Consumes 9 | import javax.ws.rs.POST 10 | import javax.ws.rs.Path 11 | import javax.ws.rs.Produces 12 | import javax.ws.rs.core.Response 13 | 14 | import static cd.pipeline.listen.core.git.GitUtils.toBranchName 15 | import static javax.ws.rs.core.MediaType.APPLICATION_JSON 16 | import static javax.ws.rs.core.Response.Status.BAD_REQUEST 17 | 18 | @Slf4j 19 | @Path("/providers/gitlab") 20 | @Produces(APPLICATION_JSON) 21 | @Consumes(APPLICATION_JSON) 22 | class GitLabWebHookResource { 23 | 24 | private final EventBus bus 25 | 26 | GitLabWebHookResource(EventBus bus) { 27 | this.bus = bus 28 | } 29 | 30 | @POST 31 | Response trigger(String payload) { 32 | def slurper = new JsonSlurper() 33 | def info = slurper.parseText(payload) 34 | try { 35 | verifyRequest(info) 36 | } catch (AssertionError e) { 37 | log.warn("Received GitLab WebHook payload with unexpected format:\n{}", e.message) 38 | return Response.status(BAD_REQUEST.statusCode).build() 39 | } 40 | return handleRequest(info) 41 | } 42 | 43 | private Response handleRequest(request) { 44 | def event = createGitTriggerEvent(request) 45 | bus.post(event) 46 | Response.noContent().build() 47 | } 48 | 49 | private createGitTriggerEvent(request) { 50 | def url = request.repository.url 51 | def branch = toBranchName(request.ref) 52 | new GitTriggerEvent(url, branch) 53 | } 54 | 55 | private verifyRequest(request) throws AssertionError { 56 | assert request.commits 57 | assert request.user_name 58 | assert request.ref 59 | assert request.repository 60 | assert request.repository.name 61 | assert request.repository.homepage 62 | assert request.repository.url 63 | assertValidUri(request.repository.url) 64 | } 65 | 66 | private assertValidUri(String uri) throws AssertionError { 67 | assert uri.contains('@') 68 | def checkUri = uri.substring(uri.indexOf('@') + 1) 69 | try { 70 | new URI(checkUri) 71 | } catch (URISyntaxException e) { 72 | throw new AssertionError(e.message, e) 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /subprojects/groovydsl/src/main/groovy/cd/pipeline/messenger/internal/EmailMessenger.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.messenger.internal; 2 | 3 | import cd.pipeline.event.PipeEvent; 4 | import cd.pipeline.messenger.MessageContext; 5 | import cd.pipeline.messenger.SpecializedMessenger; 6 | import org.springframework.mail.javamail.JavaMailSender; 7 | import org.springframework.mail.javamail.MimeMessageHelper; 8 | 9 | import javax.mail.MessagingException; 10 | import javax.mail.internet.MimeMessage; 11 | 12 | public class EmailMessenger implements SpecializedMessenger { 13 | private final JavaMailSender sender; 14 | private final String from; 15 | 16 | public EmailMessenger(JavaMailSender sender, String from) { 17 | this.sender = sender; 18 | this.from = from; 19 | } 20 | 21 | public boolean accepts(MessageContext context) { 22 | return context instanceof EmailContext; 23 | } 24 | 25 | public void process(MessageContext genericContext, PipeEvent event) { 26 | EmailContext context = (EmailContext) genericContext; 27 | 28 | StringBuilder subject = new StringBuilder(); 29 | subject.append("[pipeline] "); 30 | subject.append(event.getName()); 31 | subject.append(": "); 32 | subject.append(event.getStatus()); 33 | 34 | StringBuilder body = new StringBuilder(); 35 | body.append("Pipeline Name: "); 36 | body.append(event.getName()); 37 | body.append(System.getProperty("line.separator")); 38 | body.append("Status: "); 39 | body.append(event.getStatus()); 40 | body.append(System.getProperty("line.separator")); 41 | body.append("Description: "); 42 | body.append(event.getDescription()); 43 | body.append(System.getProperty("line.separator")); 44 | body.append("Details:"); 45 | body.append(System.getProperty("line.separator")); 46 | body.append(event.getDetails()); 47 | 48 | MimeMessage msg = sender.createMimeMessage(); 49 | MimeMessageHelper helper = new MimeMessageHelper(msg); 50 | try { 51 | helper.setFrom(from); 52 | for (String address : context.getTo()) { 53 | helper.addTo(address); 54 | } 55 | for (String address : context.getCc()) { 56 | helper.addCc(address); 57 | } 58 | helper.setSubject(subject.toString()); 59 | helper.setText(body.toString()); 60 | } catch (MessagingException e) { 61 | throw new RuntimeException(e); 62 | } 63 | sender.send(msg); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 12 | set DEFAULT_JVM_OPTS= 13 | 14 | set DIRNAME=%~dp0 15 | if "%DIRNAME%" == "" set DIRNAME=. 16 | set APP_BASE_NAME=%~n0 17 | set APP_HOME=%DIRNAME% 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windowz variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | if "%@eval[2+2]" == "4" goto 4NT_args 53 | 54 | :win9xME_args 55 | @rem Slurp the command line arguments. 56 | set CMD_LINE_ARGS= 57 | set _SKIP=2 58 | 59 | :win9xME_args_slurp 60 | if "x%~1" == "x" goto execute 61 | 62 | set CMD_LINE_ARGS=%* 63 | goto execute 64 | 65 | :4NT_args 66 | @rem Get arguments from the 4NT Shell from JP Software 67 | set CMD_LINE_ARGS=%$ 68 | 69 | :execute 70 | @rem Setup the command line 71 | 72 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 73 | 74 | @rem Execute Gradle 75 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 76 | 77 | :end 78 | @rem End local scope for the variables with windows NT shell 79 | if "%ERRORLEVEL%"=="0" goto mainEnd 80 | 81 | :fail 82 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 83 | rem the _cmd.exe /c_ return code! 84 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 85 | exit /b 1 86 | 87 | :mainEnd 88 | if "%OS%"=="Windows_NT" endlocal 89 | 90 | :omega 91 | -------------------------------------------------------------------------------- /subprojects/groovydsl/src/main/groovy/cd/pipeline/dsl/internal/DefaultPipelineDsl.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.dsl.internal; 2 | 3 | import cd.pipeline.api.Pipeline; 4 | import cd.pipeline.config.DefaultPipeline; 5 | import cd.pipeline.dsl.AnnounceDsl; 6 | import cd.pipeline.dsl.MessengerDsl; 7 | import cd.pipeline.dsl.StageDsl; 8 | import cd.pipeline.event.PipeEvent; 9 | import cd.pipeline.messenger.MessageContext; 10 | import cd.pipeline.util.ConfigureUtil; 11 | import groovy.lang.Closure; 12 | 13 | import java.util.ArrayList; 14 | import java.util.LinkedHashSet; 15 | import java.util.List; 16 | import java.util.Set; 17 | 18 | public class DefaultPipelineDsl implements InternalPipelineDsl { 19 | private final AnnounceDsl announce = new DefaultAnnounceDsl(); 20 | private final MessengerDsl messenger = new DefaultMessengerDsl(); 21 | private Set stages = new LinkedHashSet<>(); 22 | private List events = new ArrayList<>(); 23 | private boolean hasFailedStage = false; 24 | 25 | @Override 26 | public StageDsl stage(final String name, final Closure closure) { 27 | final InternalStageDsl stage = new DefaultStageDsl(name); 28 | if (hasFailedStage) { 29 | return stage; 30 | } 31 | try { 32 | ConfigureUtil.configure(stage, closure); 33 | stages.add(stage); 34 | events.add(new PipeEvent("ProjectName", "Success", stage.getDescription(), "")); 35 | return stage; 36 | } catch (PipelineException e) { 37 | hasFailedStage = true; 38 | events.add(new PipeEvent("ProjectName", "Failure", stage.getDescription(), e.getMessage())); 39 | return stage; 40 | } 41 | } 42 | 43 | @Override 44 | public void echo(final Object value) { 45 | System.out.print(value); 46 | } 47 | 48 | @Override 49 | public void echo(final String format, final Object... values) { 50 | System.out.printf(format, values); 51 | } 52 | 53 | @Override 54 | public Pipeline export() { 55 | final Pipeline pipeline = new DefaultPipeline(); 56 | for (InternalStageDsl stage : stages) { 57 | pipeline.add(stage.export()); 58 | } 59 | return pipeline; 60 | } 61 | 62 | @Override 63 | public AnnounceDsl announce(Closure closure) { 64 | ConfigureUtil.configure(announce, closure); 65 | for (MessageContext context : announce.toContexts()) { 66 | messenger.toMessenger().process(context, events.get(events.size() - 1)); 67 | } 68 | return announce; 69 | } 70 | 71 | @Override 72 | public MessengerDsl messenger(Closure closure) { 73 | ConfigureUtil.configure(messenger, closure); 74 | return messenger; 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /subprojects/listener/src/main/groovy/cd/pipeline/listen/resources/GitHubWebHookResource.groovy: -------------------------------------------------------------------------------- 1 | package cd.pipeline.listen.resources 2 | 3 | import cd.pipeline.listen.core.GitTriggerEvent 4 | import com.google.common.eventbus.EventBus 5 | import groovy.json.JsonSlurper 6 | import groovy.util.logging.Slf4j 7 | 8 | import javax.ws.rs.* 9 | import javax.ws.rs.core.Response 10 | 11 | import static cd.pipeline.listen.core.git.GitUtils.toBranchName 12 | import static cd.pipeline.listen.core.git.GithubUtils.toGithubSshUrl 13 | import static javax.ws.rs.core.MediaType.APPLICATION_FORM_URLENCODED 14 | import static javax.ws.rs.core.MediaType.APPLICATION_JSON 15 | import static javax.ws.rs.core.Response.Status.BAD_REQUEST 16 | 17 | @Slf4j 18 | @Path("/providers/github") 19 | @Produces(APPLICATION_JSON) 20 | @Consumes(APPLICATION_FORM_URLENCODED) 21 | class GitHubWebHookResource { 22 | 23 | private final EventBus bus 24 | 25 | GitHubWebHookResource(EventBus bus) { 26 | this.bus = bus 27 | } 28 | 29 | @POST 30 | Response trigger(@FormParam('payload') String payload) { 31 | def slurper = new JsonSlurper() 32 | def info = slurper.parseText(payload) 33 | try { 34 | verifyRequest(info) 35 | } catch (AssertionError e) { 36 | log.warn("Received GitHub WebHook payload with unexpected format:\n{}", e.message) 37 | return Response.status(BAD_REQUEST.statusCode).build() 38 | } 39 | return handleRequest(info) 40 | } 41 | 42 | private Response handleRequest(request) { 43 | def event = createGitTriggerEvent(request) 44 | bus.post(event) 45 | Response.noContent().build() 46 | } 47 | 48 | private createGitTriggerEvent(request) { 49 | def url = toGithubSshUrl(request.repository.url) 50 | def branch = toBranchName(request.ref) 51 | new GitTriggerEvent(url, branch) 52 | } 53 | 54 | private verifyRequest(request) throws AssertionError { 55 | assert request.commits 56 | assert request.head_commit 57 | assert request.pusher 58 | assert request.pusher.email 59 | assert request.pusher.name 60 | assert request.ref 61 | assert request.repository 62 | assert request.repository.name 63 | assert request.repository.master_branch 64 | assert request.repository.language 65 | assert request.repository.private == true || 66 | request.repository.private == false 67 | assert request.repository.url 68 | assertValidUri(request.repository.url) 69 | } 70 | 71 | private assertValidUri(String uri) throws AssertionError { 72 | try { 73 | new URI(uri) 74 | } catch (URISyntaxException e) { 75 | throw new AssertionError(e.message, e) 76 | } 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /subprojects/runner/src/main/groovy/cd/pipeline/runner/cli/Main.groovy: -------------------------------------------------------------------------------- 1 | package cd.pipeline.runner.cli 2 | 3 | import cd.pipeline.runner.cli.command.* 4 | import cd.pipeline.runner.internal.ServiceRegistry 5 | import cd.pipeline.util.ServiceLocator 6 | import com.beust.jcommander.JCommander 7 | import com.beust.jcommander.ParameterException 8 | import org.pipelinelabs.pipeline.runner.cli.command.* 9 | 10 | class Main { 11 | static final int EXIT_FAILURE = 1 12 | static final int EXIT_SUCCESS = 0 13 | PrintStream outputConsumer = System.out 14 | PrintStream errorConsumer = System.err 15 | 16 | int run(String programName, String... args) { 17 | final mainOptions = new MainCommand() 18 | final JCommander cli = createCommander(programName, mainOptions) 19 | final helpCmd = new HelpCommand(errorConsumer, cli) 20 | final List commands = new ArrayList<>() 21 | commands.add(helpCmd) 22 | commands.addAll(createCommands()) 23 | commands.each { cmd -> 24 | cli.addCommand(cmd) 25 | } 26 | 27 | final cmdName 28 | final handledCommand 29 | try { 30 | cli.parse(args) 31 | cmdName = cli.getParsedCommand() 32 | handledCommand = handleCommand(mainOptions, helpCmd, commands, cmdName) 33 | } catch (ParameterException e) { 34 | errorConsumer.println(e.getMessage()) 35 | helpCmd.outputUsage() 36 | return EXIT_FAILURE 37 | } 38 | if (handledCommand) { 39 | return EXIT_SUCCESS 40 | } 41 | return EXIT_FAILURE 42 | } 43 | 44 | private boolean handleCommand(final MainCommand mainOptions, final HelpCommand helpCmd, 45 | final List commands, final String cmdName) { 46 | if (mainOptions.help) { 47 | helpCmd.outputUsage() 48 | return false 49 | } 50 | final Command cmd = commands.find { it.canHandle(cmdName) } 51 | if (cmd) { 52 | cmd.handle() 53 | return true 54 | } 55 | helpCmd.outputUsage() 56 | return false 57 | } 58 | 59 | private List createCommands() { 60 | final List commands = new ArrayList<>() 61 | commands.add(new RunCommand(outputConsumer, getRegistry())) 62 | commands.add(new FeedbackCommand(outputConsumer)) 63 | return commands 64 | } 65 | 66 | private JCommander createCommander(final String name, final MainCommand mainOptions) { 67 | final cli = new JCommander(mainOptions); 68 | cli.with { 69 | programName = name 70 | allowAbbreviatedOptions = true 71 | } 72 | return cli 73 | } 74 | 75 | private ServiceRegistry getRegistry() { 76 | return new ServiceLocator().find(ServiceRegistry) 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /subprojects/runner/src/test/groovy/cd/pipeline/runner/cli/dsl/PipelineScriptRunnerSpec.groovy: -------------------------------------------------------------------------------- 1 | package cd.pipeline.runner.cli.dsl 2 | 3 | import cd.pipeline.runner.internal.DefaultServiceRegistry 4 | import spock.lang.Ignore 5 | import spock.lang.Specification 6 | 7 | import static org.hamcrest.CoreMatchers.equalTo 8 | import static org.hamcrest.CoreMatchers.is 9 | import static org.hamcrest.Matchers.containsString 10 | import static spock.util.matcher.HamcrestSupport.that 11 | 12 | class PipelineScriptRunnerSpec extends Specification { 13 | 14 | @Ignore 15 | def "Cannot call pipeline methods from the stage closure"() { 16 | def script = """ 17 | stage 'commit', { 18 | stage 'stageFromAStage', { 19 | 20 | } 21 | } 22 | """ 23 | when: 24 | runScript(script) 25 | 26 | then: 27 | thrown(MissingMethodException) 28 | } 29 | 30 | def "Can use build-in groovy println"() { 31 | def script = """ 32 | println 'Hello World' 33 | """ 34 | 35 | expect: 36 | def output = runScript(script) 37 | that output, is(equalTo('Hello World\n')) 38 | } 39 | 40 | def "Can execute stage with build-in groovy println"() { 41 | def script = """ 42 | stage 'hello', { 43 | println 'Hello Using my own binding' 44 | } 45 | """ 46 | 47 | expect: 48 | def output = runScript(script) 49 | that output, is(equalTo('Hello Using my own binding\n')) 50 | } 51 | 52 | def "Can execute self-defined echo"() { 53 | def script = """ 54 | echo 'Hello %s', 'World' 55 | """ 56 | 57 | expect: 58 | def output = runScript(script) 59 | that output, is(equalTo('Hello World')) 60 | } 61 | 62 | def "Can execute stage with self-defined echo"() { 63 | def script = """ 64 | stage 'hello', { 65 | echo 'Hello %s', 'World' 66 | } 67 | """ 68 | 69 | expect: 70 | def output = runScript(script) 71 | that output, is(equalTo('Hello World')) 72 | } 73 | 74 | def "Can execute system command from stage"() { 75 | def script = """ 76 | stage 'execute dir system command', { 77 | run "dir" 78 | } 79 | """ 80 | 81 | expect: 82 | def output = runScript(script) 83 | that output, containsString('runner.gradle') 84 | } 85 | 86 | private String runScript(final String script) { 87 | def output = new ByteArrayOutputStream() 88 | def stream = new PrintStream(output) 89 | def registry = new DefaultServiceRegistry() 90 | final PipelineScriptRunner buildScript = new PipelineScriptRunner(registry, stream, script) 91 | 92 | buildScript.run() 93 | 94 | return output.toString() 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /subprojects/api/src/main/groovy/cd/pipeline/util/ServiceLocator.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.util; 2 | 3 | import java.util.HashSet; 4 | import java.util.Iterator; 5 | import java.util.ServiceLoader; 6 | import java.util.Set; 7 | 8 | /** 9 | * Locate services. 10 | *

11 | * Services are located via the standard Java 6+ way of discovering services, 12 | * as done by the {@link ServiceLoader}. 13 | * 14 | * @see ServiceLoader 15 | */ 16 | public class ServiceLocator { 17 | 18 | /** 19 | * Find an implementation for the given class type. 20 | * 21 | * @param clazz Class type to find an implementation for 22 | * @return Instance of the found implementation 23 | * @throws ServiceLookupException When no implementation is found, 24 | * or when multiple implementations are found 25 | */ 26 | public T find(final Class clazz) throws ServiceLookupException { 27 | final Set found = findAll(clazz); 28 | if (found.isEmpty()) { 29 | throw new NoServiceImplementationFound(clazz); 30 | } 31 | if (found.size() != 1) { 32 | throw new MultipleServiceImplementationsFound(clazz); 33 | } 34 | return found.iterator().next(); 35 | } 36 | 37 | /** 38 | * Find all available implementations for the given class type. 39 | * 40 | * @param clazz Class type to find an implementation for 41 | * @return Set of instances of the found implementations 42 | */ 43 | public Set findAll(final Class clazz) { 44 | final ServiceLoader loader = ServiceLoader.load(clazz); 45 | return newSet(loader.iterator()); 46 | } 47 | 48 | private Set newSet(final Iterator it) { 49 | final HashSet set = new HashSet<>(); 50 | while (it.hasNext()) { 51 | set.add(it.next()); 52 | } 53 | return set; 54 | } 55 | 56 | private class NoServiceImplementationFound extends ServiceLookupException { 57 | 58 | private Class clazz; 59 | 60 | NoServiceImplementationFound(final Class clazz) { 61 | this.clazz = clazz; 62 | } 63 | 64 | @Override 65 | public String getMessage() { 66 | final String msg = "Found no implementation for '%s'"; 67 | return String.format(msg, clazz.getSimpleName()); 68 | } 69 | } 70 | 71 | private class MultipleServiceImplementationsFound extends ServiceLookupException { 72 | 73 | private Class clazz; 74 | 75 | MultipleServiceImplementationsFound(final Class clazz) { 76 | this.clazz = clazz; 77 | } 78 | 79 | @Override 80 | public String getMessage() { 81 | final String msg = "Found multiple implementations for '%s', expected only one implementation"; 82 | return String.format(msg, clazz.getSimpleName()); 83 | } 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /subprojects/runner/src/test/groovy/cd/pipeline/runner/cli/MainSpec.groovy: -------------------------------------------------------------------------------- 1 | package cd.pipeline.runner.cli 2 | 3 | import spock.lang.Specification 4 | import spock.lang.Unroll 5 | 6 | class MainSpec extends Specification { 7 | 8 | static final int EXIT_SUCCESS = 0 9 | static final int EXIT_FAILURE = 1 10 | def programName = 'test' 11 | def stdout = new ByteArrayOutputStream() 12 | def errout = new ByteArrayOutputStream() 13 | def main = getTestableMain() 14 | 15 | @Unroll 16 | def "Can execute main with argument(s) #args to get help info with an exitStatus of '#expectedExitStatus'"() { 17 | def actualExistStatus = main.run(programName, args as String[]) 18 | 19 | expect: 20 | assert stdout.toString().isEmpty() 21 | assert errout.toString().contains('--help, -h') 22 | assert errout.toString().contains('help Show help information about a command') 23 | assert actualExistStatus == expectedExitStatus 24 | 25 | where: 26 | args || expectedExitStatus 27 | [''] || EXIT_FAILURE 28 | ['-h'] || EXIT_FAILURE 29 | ['--help'] || EXIT_FAILURE 30 | ['help'] || EXIT_SUCCESS 31 | } 32 | 33 | @Unroll 34 | def "Can give positive feedback with command '#cmd' and option '#option'"() { 35 | def actualExistStatus = main.run(programName, cmd, option, 'did good') 36 | 37 | expect: 38 | assert errout.toString().isEmpty() 39 | assert stdout.toString().equals('Thnx, received your positive feedback: did good\n') 40 | assert actualExistStatus == EXIT_SUCCESS 41 | 42 | where: 43 | cmd | option 44 | 'feedback' | '-p' 45 | 'feedback' | '--positive' 46 | } 47 | 48 | @Unroll 49 | def "Can give negative feedback with command '#cmd' and option '#option'"() { 50 | def actualExistStatus = main.run(programName, cmd, option, 'did bad') 51 | 52 | expect: 53 | assert errout.toString().isEmpty() 54 | assert stdout.toString().equals('Thnx, received your negative feedback: did bad\n') 55 | assert actualExistStatus == EXIT_SUCCESS 56 | 57 | where: 58 | cmd | option 59 | 'feedback' | '-n' 60 | 'feedback' | '--negative' 61 | } 62 | 63 | def "Can run Pipeline from DSL script file"() { 64 | def actualExistStatus = main.run(programName, 'run', './src/test/resources/helloworld.txt') 65 | 66 | expect: 67 | assert errout.toString().isEmpty() 68 | assert stdout.toString().contains('Hello World') 69 | assert actualExistStatus == EXIT_SUCCESS 70 | } 71 | 72 | def "Throws an IllegalArgumentException if DSL script file does not exist"() { 73 | when: 74 | main.run(programName, 'run', 'notexisting/notexisting.txt') 75 | 76 | then: 77 | thrown(IllegalArgumentException) 78 | } 79 | 80 | private Main getTestableMain() { 81 | def main = new Main() 82 | main.outputConsumer = new PrintStream(stdout) 83 | main.errorConsumer = new PrintStream(errout) 84 | return main 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /subprojects/groovydsl/src/main/groovy/cd/pipeline/dsl/internal/DefaultStageDsl.java: -------------------------------------------------------------------------------- 1 | package cd.pipeline.dsl.internal; 2 | 3 | import cd.pipeline.api.Stage; 4 | import cd.pipeline.config.DefaultStage; 5 | import org.apache.maven.shared.utils.cli.CommandLineException; 6 | import org.apache.maven.shared.utils.cli.CommandLineUtils; 7 | import org.apache.maven.shared.utils.cli.Commandline; 8 | import org.apache.maven.shared.utils.cli.WriterStreamConsumer; 9 | 10 | import java.io.BufferedWriter; 11 | import java.io.OutputStreamWriter; 12 | import java.io.PrintStream; 13 | import java.util.LinkedList; 14 | import java.util.List; 15 | 16 | public class DefaultStageDsl implements InternalStageDsl { 17 | private final String name; 18 | private String description; 19 | private List tasks = new LinkedList<>(); 20 | 21 | public DefaultStageDsl(final String name) { 22 | this.name = name; 23 | } 24 | 25 | @Override 26 | public String getName() { 27 | return name; 28 | } 29 | 30 | @Override 31 | public String getDescription() { 32 | return description; 33 | } 34 | 35 | @Override 36 | public void setDescription(final String description) { 37 | this.description = description; 38 | } 39 | 40 | @Override 41 | public void run(final String command) throws Exception { 42 | tasks.add(new ShellCommandTaskDsl(command)); 43 | 44 | final ExternalProcess process = new ExternalProcess(); 45 | process.setCommand(command); 46 | process.run(); 47 | } 48 | 49 | @Override 50 | public Stage export() { 51 | final Stage stage = new DefaultStage(); 52 | for (InternalTaskDsl task : tasks) { 53 | stage.add(task.export()); 54 | } 55 | return stage; 56 | } 57 | 58 | private class ExternalProcess { 59 | 60 | private PrintStream out = System.out; 61 | private PrintStream err = System.err; 62 | private String command; 63 | 64 | public void setCommand(final String command) { 65 | this.command = command; 66 | } 67 | 68 | public void run() { 69 | out.println(); 70 | final Commandline cl = new Commandline(command); 71 | final int exitStatus = executeCommand(cl, createStreamingConsumer(out), createStreamingConsumer(err)); 72 | if (0 != exitStatus) { 73 | final String msg = "Error occurred during execution of command [%s]"; 74 | throw new PipelineException(String.format(msg, command)); 75 | } 76 | } 77 | 78 | private int executeCommand(final Commandline cl, 79 | final WriterStreamConsumer stdOut, final WriterStreamConsumer stdErr) { 80 | final int exitStatus; 81 | try { 82 | exitStatus = CommandLineUtils.executeCommandLine(cl, stdOut, stdErr); 83 | } catch (CommandLineException e) { 84 | throw new PipelineException("Failed to execute command", e); 85 | } 86 | return exitStatus; 87 | } 88 | 89 | private WriterStreamConsumer createStreamingConsumer(final PrintStream stream) { 90 | return new WriterStreamConsumer(new BufferedWriter(new OutputStreamWriter(stream))); 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /docs/User-Guide-Set-up-with-Central-Repository.md: -------------------------------------------------------------------------------- 1 | Using Pipeline with a Central Repository 2 | =============== 3 | _Currently only git-based repositories are supported_ 4 | When a user pushes to a central code repository like git, a pipeline is triggered based on the configuration that is located within the that git repository. 5 | 6 | ``` 7 | +----------+ +----------+ 8 | | User | | | 9 | | | +----> | git repo | 10 | | git push | | | 11 | +----------+ +-+--------+ 12 | | 13 | | git post-hook calls 14 | | via web-call 15 | | 16 | +------------------------|-----------+ 17 | | | | 18 | | v | 19 | | +----------+ +----------+ | 20 | | | Pipeline |<------| Pipeline | | 21 | | | Runner | | Listener | | 22 | | +----+-----+ +----------+ | 23 | | | | 24 | | v | 25 | | Sent status | 26 | | email | 27 | | +------------------+ 28 | | |Runs on one server| 29 | +-----------------+------------------+ 30 | ``` 31 | The simple set-up requires that the central code repository supports post-hooks as these are executed after code is committed/pushed to the repository. Git is such type of code repository. 32 | 33 | Requirements 34 | ============ 35 | - A Linux-based server to run Pipeline on 36 | **Note:** This server must be accessible via HTTP(S) from your central code repository 37 | - Java 7+ 38 | - Git _(as currently only git is supported)_ 39 | - _(Windows platform currently not tested, therefor may or may not work)_ 40 | - A central code repository that supports post-hooks eg. 41 | - [Github](http://github.com) 42 | - [GitLab](http://gitlab.org) 43 | - Any tools your code requires for compiling, testing, etc (its pipeline) 44 | **Note:** When working with Java code, it must currently support Java 7 as Pipeline itself and your project pipeline will use the same Java runtime 45 | 46 | Installation 47 | ============ 48 | Installing Pipeline for this simple set-up is done as follows: 49 | 50 | 1. Download the Pipeline installation package 51 | 2. Unpack the installation package to a location of choice 52 | 3. Add `{unpack-location}/bin/` to your environment `PATH` variable 53 | 4. Start pipeline by executing the command `pipe-listen server`, the log output will tell that the server is running on port 8080 54 | 5. Register the Pipeline post-hook with the central code repository: 55 | - For Github this means, configure the Pipeline post hook url as `Service Hooks > Web Hook URLS`: 56 | [http://{ip}:8080/providers/github](http://{ip}:8080/providers/github) 57 | - For GitLab this means, configure the Pipeline post hook url as `Settings > Web Hooks`: 58 | [http://{ip}:8080/providers/gitlab](http://{ip}:8080/providers/gitlab) 59 | 6. That's it! On your next push to the central code repository, your pipeline should be triggered 60 | 61 | Now, add a pipeline configuration to your project! 62 | 63 | Configuration 64 | ============= 65 | 1. In the root of your code repository, create a `project.pipe` script 66 | 2. Configure the pipeline. 67 | - **Note:** Configuration is explained on [project pipeline configuration](User-Guide-Project-Pipeline-Configuration) -------------------------------------------------------------------------------- /docs/User-Guide-Project-Pipeline-Configuration.md: -------------------------------------------------------------------------------- 1 | # Configuring the Pipeline Configuration for your project 2 | 3 | The configuration of a project pipeline is stored in the file `project.pipe` in the root of the code repository of the project, and versioned together with the code. When referring to a pipeline, we talk about the configuration within the `project.pipe` file. 4 | 5 | Currently the following configuration is supported in the pipeline: 6 | ```groovy 7 | stage {NAME}, { 8 | description = '{DESC}' 9 | run '{COMMAND}' 10 | } 11 | ``` 12 | This defines one step in the pipeline, which is called a "stage". 13 | Replace `NAME` with a simple name of the stage, `DESC` can be a more descriptive description of this stage (even multi-line), and `COMMAND` should be the command to execute during this stage. 14 | 15 | The pipeline can exist of multiple stages, which are executed in order of configuring. 16 | When a stage command exits with an error status, the whole pipeline is stopped and set as failed. 17 | 18 | As there's no GUI of any kind to Pipeline, the way to get notified of failed or succeeding pipeline runs is (currently) by email. To get notifications, add the following configuration to the pipeline: 19 | 20 | ```groovy 21 | messenger { 22 | email { 23 | host = 'smtp.gmail.com' 24 | port = 587 25 | tls = true 26 | username = 'user@site.com' 27 | password = 'password' 28 | from = 'from@site.org' 29 | } 30 | } 31 | 32 | announce { 33 | email { 34 | to 'EMAIL' 35 | } 36 | } 37 | ``` 38 | Replace `EMAIL` with the emailaddress to sent notifications to. 39 | 40 | > **WARNING** 41 | > Don't store your email server password in a __public__ git repository. Even in a __private__ repository be warned that this password will stay in your history, so for collaborators or when the repository eventually gets public, can recover your password from the git history. See Github' [Remove sensitive data](https://help.github.com/articles/remove-sensitive-data) article for details. 42 | > 43 | > The email functionality is **NOT** required, if you don't add the messenger and announce sections to your pipeline configuration, the pipeline will work just fine. Without it, the only way to see if the pipeline succeeded/failed is in the pipe-listen log. 44 | __(in the future there will be a secure solution for this)__ 45 | 46 | Example pipeline 47 | ================ 48 | An example pipeline configuration could look as following: 49 | 50 | ```groovy 51 | messenger { 52 | email { 53 | host = 'smtp.gmail.com' 54 | port = 587 55 | tls = true 56 | username = 'user@site.com' 57 | password = 'password' 58 | from = 'from@site.org' 59 | } 60 | } 61 | 62 | stage commit, { 63 | description = 'Compile and test' 64 | run 'mvn test' 65 | } 66 | 67 | stage component-test, { 68 | description = 'Run component tests' 69 | run 'mvn verify -DskipTests=true -DskipITs=false' 70 | } 71 | 72 | stage document, { 73 | description = 'Generate documentation' 74 | run 'mvn -P generate-docs' 75 | } 76 | 77 | stage inspect, { 78 | description = 'Inspect the quality' 79 | run 'mvn sonar:sonar' 80 | } 81 | 82 | stage snapshot-release, { 83 | description = 'Do a snapshot release' 84 | run 'mvn deploy' 85 | } 86 | 87 | announce { 88 | email { 89 | to 'list@mailing.org' 90 | } 91 | } 92 | ``` 93 | Which will execute the following commands in order: 94 | - `mvn test` 95 | - `mvn verify -DskipTests=true -DskipITs=false` 96 | - `mvn -P generate-docs` 97 | - `mvn sonar:sonar` 98 | - `mvn deploy` 99 | - sent email to `list@mailing.org` notifying about the pipeline status -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Pipeline 2 | ======== 3 | 4 | **THIS PROJECT IS ABANDONED** 5 | 6 | [Pipeline](http://pipeline.cd) is a Continuous Delivery (CD) tool, which we and all stakeholders of the CD pipeline love to use. Licensed under the [MIT License][0] 7 | 8 | Find the latest release on the [releases][6] page. And follow the [installation][8] and [configuration][9] documentation to get started. 9 | 10 | - [Website](http://pipeline.cd) 11 | - [Source](http://github.pipeline.cd) 12 | - [Documentation](http://docs.pipeline.cd) 13 | - [User/Dev Forum][3] 14 | - [Issue Tracker](http://issues.pipeline.cd) 15 | - Ignite talk "Continuous Integration and Delivery tools, do we really like using them?" 16 | - [Preparation run recording][4] 17 | - [@DevOpsDays Amsterdam, June 14th 2013 recording][5] 18 | 19 | Follow us on Twitter [@pipelinecd](https://twitter.com/pipelinecd) 20 | 21 | State of the project 22 | -------------------- 23 | ~~This project is currently in a very early state of development. Development happens in a LEAN way, bit by bit adding features and improving based on user experiences and feedback. Give your feedback at the [user/dev forum][3].~~ 24 | 25 | **THIS PROJECT IS ABANDONED** 26 | 27 | Features 28 | -------- 29 | - From the ground-up, pipeline-based 30 | - Pipeline configuration in pipeline domain language (DSL) 31 | - Version controlled pipeline configuration 32 | - Support for public and private git repositories 33 | - hosted on [Github](http://github.com), triggered via [webhook][7] 34 | - hosted on [GitLab(.org/com)](http://gitlab.org), triggered via [webhook] as explained on [Web hooks] help page of your GitLab installation 35 | 36 | Roadmap 37 | ------- 38 | Some ideas on epics/topics we want to have implemented along the way, in no particular order: 39 | 40 | - Configuration service that provides generic configuration to all pipelines 41 | - Also supporting a secure way for sensitive configuration data 42 | - Native packages for different OS'es, for easy installation 43 | - Also providing scripts to run services as system daemons/services 44 | - Extensive logging and monitoring functionality 45 | - Extend pipeline configuration with environment and execution information for simpler maintenance by not depending on specific server configurations, the pipeline takes care of everything in a versioned manner. For example: 46 | - Environment variables to make available to the whole pipeline, specific stages or specific commands 47 | - System applications and versions which are required to run the pipeline are installed and configured automatically 48 | - ... 49 | - Different GUI's, for the different types of users and usages 50 | - All communicating with one single API against the system 51 | - Support for different authentication and authorization schemes 52 | 53 | Building Pipeline 54 | -------------- 55 | The only prerequisite is that you have JDK 7 or higher installed. 56 | 57 | After cloning the project, type `./gradlew build` (Windows: `gradlew build`). All build dependencies, 58 | including [Gradle](http://www.gradle.org) itself, will be downloaded automatically (unless already present). 59 | 60 | Contributing to this project 61 | ---------------------------- 62 | 63 | Anyone and everyone is welcome to contribute. Please take a moment to 64 | review the [guidelines for contributing](CONTRIBUTING.md). 65 | 66 | - [Bug reports](CONTRIBUTING.md#bugs) 67 | - [Feature requests](CONTRIBUTING.md#features) 68 | - [Pull requests](CONTRIBUTING.md#pull-requests) 69 | 70 | Special thanks to 71 | ----------------- 72 | - [Andrew Oberstar](http://www.andrewoberstar.com) for being the very first code contributor and all-round providing great support and feedback 73 | - [Martin Kovachki](http://www.linkedin.com/in/martinkov) for awesome logo designs 74 | - [Tricode](http://www.tricode.nl) for sponsoring developer-time and resources in 2013 75 | 76 | [0]: http://github.pipeline.cd/blob/master/LICENSE 77 | [3]: http://forum.pipeline.cd 78 | [4]: http://www.youtube.com/watch?v=shF_v5shzjU 79 | [5]: http://www.youtube.com/watch?v=-StobwMgRNE 80 | [6]: https://github.com/pipelinecd/pipeline/releases 81 | [7]: https://help.github.com/articles/post-receive-hooks 82 | [8]: http://docs.pipeline.cd/User-Guide%3A-Set-up-with-Central-Repository 83 | [9]: http://docs.pipeline.cd/User-Guide%3A-Project-Pipeline-Configuration 84 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Pipeline 2 | 3 | Looking to contribute something to Pipeline? Here's how you can help. 4 | 5 | 6 | ## Bugs reports 7 | 8 | A bug is a _demonstrable problem_ that is caused by the code in the 9 | repository. Good bug reports are extremely helpful – thank you! 10 | 11 | Guidelines for bug reports: 12 | 13 | 1. **Use the GitHub issue search** — check if the issue has already been 14 | reported. 15 | 16 | 2. **Check if the issue has been fixed** — try to reproduce it using the 17 | latest `master` or development branch in the repository. 18 | 19 | 3. **Isolate the problem** — ideally create a reduced test 20 | case and a live example. 21 | 22 | 4. Please try to be as detailed as possible in your report. Include specific 23 | information about the environment – operating system and version, browser 24 | and version, version of Pipeline – and steps required to reproduce the issue. 25 | 26 | 27 | ## Feature requests & contribution enquiries 28 | 29 | Feature requests are welcome. But take a moment to find out whether your idea 30 | fits with the scope and aims of the project. It's up to *you* to make a strong 31 | case for the inclusion of your feature. Please provide as much detail and 32 | context as possible. 33 | 34 | Contribution enquiries should take place before any significant pull request, 35 | otherwise you risk spending a lot of time working on something that we might 36 | have good reasons for rejecting. 37 | 38 | 39 | ## Pull requests 40 | 41 | Good pull requests – patches, improvements, new features – are a fantastic 42 | help. They should remain focused in scope and avoid containing unrelated 43 | commits. 44 | 45 | Make sure to adhere to the coding conventions used throughout the codebase 46 | (indentation, accurate comments, etc.) and any other requirements (such as test 47 | coverage). 48 | 49 | Please follow this process; it's the best way to get your work included in the 50 | project: 51 | 52 | 1. [Fork](http://help.github.com/fork-a-repo/) the project, clone your fork, 53 | and configure the remotes: 54 | 55 | ```bash 56 | # Clone your fork of the repo into the current directory 57 | git clone https://github.com//pipeline 58 | # Navigate to the newly cloned directory 59 | cd 60 | # Assign the original repo to a remote called "upstream" 61 | git remote add upstream git://github.com/pipelinecd/pipeline 62 | ``` 63 | 64 | 2. If you cloned a while ago, get the latest changes from upstream: 65 | 66 | ```bash 67 | git checkout master 68 | git pull upstream master 69 | ``` 70 | 71 | 3. Install the dependencies (you must have JDK 7 or higher 72 | installed), and create a new topic branch (off the main project development 73 | branch) to contain your feature, change, or fix: 74 | 75 | ```bash 76 | git checkout -b 77 | ``` 78 | 79 | 4. Make sure to update, or add to the tests when appropriate. Patches and 80 | features will not be accepted without tests. Run `gradlew check` to check that 81 | all tests pass after you've made changes. 82 | 83 | 5. Commit your changes in logical chunks. Provide clear and explanatory commit 84 | messages. Use Git's [interactive rebase](https://help.github.com/articles/interactive-rebase) feature to tidy up 85 | your commits before making them public. 86 | 87 | 6. Locally merge (or rebase) the upstream development branch into your topic branch: 88 | 89 | ```bash 90 | git pull [--rebase] upstream master 91 | ``` 92 | 93 | 7. Push your topic branch up to your fork: 94 | 95 | ```bash 96 | git push origin 97 | ``` 98 | 99 | 8. [Open a Pull Request](https://help.github.com/articles/using-pull-requests/) 100 | with a clear title and description. 101 | 102 | 9. If you are asked to amend your changes before they can be merged in, please 103 | use `git commit --amend` (or rebasing for multi-commit Pull Requests) and 104 | force push to your remote feature branch. You may also be asked to squash 105 | commits. 106 | 107 | ## License 108 | 109 | By contributing your code, 110 | 111 | You agree to license your contribution under the terms of the [MIT License](https://github.com/pipelinecd/pipeline/blob/master/LICENSE) 112 | -------------------------------------------------------------------------------- /subprojects/listener/src/test/resources/github/samplePayload.json: -------------------------------------------------------------------------------- 1 | { 2 | "after": "1481a2de7b2a7d02428ad93446ab166be7793fbb", 3 | "before": "17c497ccc7cca9c2f735aa07e9e3813060ce9a6a", 4 | "commits": [ 5 | { 6 | "added": [ 7 | ], 8 | "author": { 9 | "email": "lolwut@noway.biz", 10 | "name": "Garen Torikian", 11 | "username": "octokitty" 12 | }, 13 | "committer": { 14 | "email": "lolwut@noway.biz", 15 | "name": "Garen Torikian", 16 | "username": "octokitty" 17 | }, 18 | "distinct": true, 19 | "id": "c441029cf673f84c8b7db52d0a5944ee5c52ff89", 20 | "message": "Test", 21 | "modified": [ 22 | "README.md" 23 | ], 24 | "removed": [ 25 | ], 26 | "timestamp": "2013-02-22T13:50:07-08:00", 27 | "url": "https://github.com/octokitty/testing/commit/c441029cf673f84c8b7db52d0a5944ee5c52ff89" 28 | }, 29 | { 30 | "added": [ 31 | ], 32 | "author": { 33 | "email": "lolwut@noway.biz", 34 | "name": "Garen Torikian", 35 | "username": "octokitty" 36 | }, 37 | "committer": { 38 | "email": "lolwut@noway.biz", 39 | "name": "Garen Torikian", 40 | "username": "octokitty" 41 | }, 42 | "distinct": true, 43 | "id": "36c5f2243ed24de58284a96f2a643bed8c028658", 44 | "message": "This is me testing the windows client.", 45 | "modified": [ 46 | "README.md" 47 | ], 48 | "removed": [ 49 | ], 50 | "timestamp": "2013-02-22T14:07:13-08:00", 51 | "url": "https://github.com/octokitty/testing/commit/36c5f2243ed24de58284a96f2a643bed8c028658" 52 | }, 53 | { 54 | "added": [ 55 | "words/madame-bovary.txt" 56 | ], 57 | "author": { 58 | "email": "lolwut@noway.biz", 59 | "name": "Garen Torikian", 60 | "username": "octokitty" 61 | }, 62 | "committer": { 63 | "email": "lolwut@noway.biz", 64 | "name": "Garen Torikian", 65 | "username": "octokitty" 66 | }, 67 | "distinct": true, 68 | "id": "1481a2de7b2a7d02428ad93446ab166be7793fbb", 69 | "message": "Rename madame-bovary.txt to words/madame-bovary.txt", 70 | "modified": [ 71 | ], 72 | "removed": [ 73 | "madame-bovary.txt" 74 | ], 75 | "timestamp": "2013-03-12T08:14:29-07:00", 76 | "url": "https://github.com/octokitty/testing/commit/1481a2de7b2a7d02428ad93446ab166be7793fbb" 77 | } 78 | ], 79 | "compare": "https://github.com/octokitty/testing/compare/17c497ccc7cc...1481a2de7b2a", 80 | "created": false, 81 | "deleted": false, 82 | "forced": false, 83 | "head_commit": { 84 | "added": [ 85 | "words/madame-bovary.txt" 86 | ], 87 | "author": { 88 | "email": "lolwut@noway.biz", 89 | "name": "Garen Torikian", 90 | "username": "octokitty" 91 | }, 92 | "committer": { 93 | "email": "lolwut@noway.biz", 94 | "name": "Garen Torikian", 95 | "username": "octokitty" 96 | }, 97 | "distinct": true, 98 | "id": "1481a2de7b2a7d02428ad93446ab166be7793fbb", 99 | "message": "Rename madame-bovary.txt to words/madame-bovary.txt", 100 | "modified": [ 101 | ], 102 | "removed": [ 103 | "madame-bovary.txt" 104 | ], 105 | "timestamp": "2013-03-12T08:14:29-07:00", 106 | "url": "https://github.com/octokitty/testing/commit/1481a2de7b2a7d02428ad93446ab166be7793fbb" 107 | }, 108 | "pusher": { 109 | "email": "lolwut@noway.biz", 110 | "name": "Garen Torikian" 111 | }, 112 | "ref": "refs/heads/master", 113 | "repository": { 114 | "created_at": 1332977768, 115 | "description": "", 116 | "fork": false, 117 | "forks": 0, 118 | "has_downloads": true, 119 | "has_issues": true, 120 | "has_wiki": true, 121 | "homepage": "", 122 | "id": 3860742, 123 | "language": "Ruby", 124 | "master_branch": "master", 125 | "name": "testing", 126 | "open_issues": 2, 127 | "owner": { 128 | "email": "lolwut@noway.biz", 129 | "name": "octokitty" 130 | }, 131 | "private": false, 132 | "pushed_at": 1363295520, 133 | "size": 2156, 134 | "stargazers": 1, 135 | "url": "https://github.com/octokitty/testing", 136 | "watchers": 1 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /subprojects/listener/src/test/resources/github/samplePayloadFromBranch.json: -------------------------------------------------------------------------------- 1 | { 2 | "after": "1481a2de7b2a7d02428ad93446ab166be7793fbb", 3 | "before": "17c497ccc7cca9c2f735aa07e9e3813060ce9a6a", 4 | "commits": [ 5 | { 6 | "added": [ 7 | ], 8 | "author": { 9 | "email": "lolwut@noway.biz", 10 | "name": "Garen Torikian", 11 | "username": "octokitty" 12 | }, 13 | "committer": { 14 | "email": "lolwut@noway.biz", 15 | "name": "Garen Torikian", 16 | "username": "octokitty" 17 | }, 18 | "distinct": true, 19 | "id": "c441029cf673f84c8b7db52d0a5944ee5c52ff89", 20 | "message": "Test", 21 | "modified": [ 22 | "README.md" 23 | ], 24 | "removed": [ 25 | ], 26 | "timestamp": "2013-02-22T13:50:07-08:00", 27 | "url": "https://github.com/octokitty/testing/commit/c441029cf673f84c8b7db52d0a5944ee5c52ff89" 28 | }, 29 | { 30 | "added": [ 31 | ], 32 | "author": { 33 | "email": "lolwut@noway.biz", 34 | "name": "Garen Torikian", 35 | "username": "octokitty" 36 | }, 37 | "committer": { 38 | "email": "lolwut@noway.biz", 39 | "name": "Garen Torikian", 40 | "username": "octokitty" 41 | }, 42 | "distinct": true, 43 | "id": "36c5f2243ed24de58284a96f2a643bed8c028658", 44 | "message": "This is me testing the windows client.", 45 | "modified": [ 46 | "README.md" 47 | ], 48 | "removed": [ 49 | ], 50 | "timestamp": "2013-02-22T14:07:13-08:00", 51 | "url": "https://github.com/octokitty/testing/commit/36c5f2243ed24de58284a96f2a643bed8c028658" 52 | }, 53 | { 54 | "added": [ 55 | "words/madame-bovary.txt" 56 | ], 57 | "author": { 58 | "email": "lolwut@noway.biz", 59 | "name": "Garen Torikian", 60 | "username": "octokitty" 61 | }, 62 | "committer": { 63 | "email": "lolwut@noway.biz", 64 | "name": "Garen Torikian", 65 | "username": "octokitty" 66 | }, 67 | "distinct": true, 68 | "id": "1481a2de7b2a7d02428ad93446ab166be7793fbb", 69 | "message": "Rename madame-bovary.txt to words/madame-bovary.txt", 70 | "modified": [ 71 | ], 72 | "removed": [ 73 | "madame-bovary.txt" 74 | ], 75 | "timestamp": "2013-03-12T08:14:29-07:00", 76 | "url": "https://github.com/octokitty/testing/commit/1481a2de7b2a7d02428ad93446ab166be7793fbb" 77 | } 78 | ], 79 | "compare": "https://github.com/octokitty/testing/compare/17c497ccc7cc...1481a2de7b2a", 80 | "created": false, 81 | "deleted": false, 82 | "forced": false, 83 | "head_commit": { 84 | "added": [ 85 | "words/madame-bovary.txt" 86 | ], 87 | "author": { 88 | "email": "lolwut@noway.biz", 89 | "name": "Garen Torikian", 90 | "username": "octokitty" 91 | }, 92 | "committer": { 93 | "email": "lolwut@noway.biz", 94 | "name": "Garen Torikian", 95 | "username": "octokitty" 96 | }, 97 | "distinct": true, 98 | "id": "1481a2de7b2a7d02428ad93446ab166be7793fbb", 99 | "message": "Rename madame-bovary.txt to words/madame-bovary.txt", 100 | "modified": [ 101 | ], 102 | "removed": [ 103 | "madame-bovary.txt" 104 | ], 105 | "timestamp": "2013-03-12T08:14:29-07:00", 106 | "url": "https://github.com/octokitty/testing/commit/1481a2de7b2a7d02428ad93446ab166be7793fbb" 107 | }, 108 | "pusher": { 109 | "email": "lolwut@noway.biz", 110 | "name": "Garen Torikian" 111 | }, 112 | "ref": "refs/heads/somebranch", 113 | "repository": { 114 | "created_at": 1332977768, 115 | "description": "", 116 | "fork": false, 117 | "forks": 0, 118 | "has_downloads": true, 119 | "has_issues": true, 120 | "has_wiki": true, 121 | "homepage": "", 122 | "id": 3860742, 123 | "language": "Ruby", 124 | "master_branch": "master", 125 | "name": "testing", 126 | "open_issues": 2, 127 | "owner": { 128 | "email": "lolwut@noway.biz", 129 | "name": "octokitty" 130 | }, 131 | "private": false, 132 | "pushed_at": 1363295520, 133 | "size": 2156, 134 | "stargazers": 1, 135 | "url": "https://github.com/octokitty/testing", 136 | "watchers": 1 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /docs/Technical-Design-DSL-Design-Ideas.md: -------------------------------------------------------------------------------- 1 | # Domain-Specific Languague (DSL) Design 2 | 3 | ## Legenda 4 | * **DSL** 5 | Domain Specific Language; 6 | * **Pipeline definition** 7 | Defines the configurable pipeline that can be reused to be run with different execution configurations; 8 | * **Execution configuration** 9 | Configuration to use when executing the pipeline definition; giving meaning to the pipeline. 10 | 11 | ## Pipeline definition 12 | The pipeline definition script is a DSL implemented in Groovy focussed on the domain of pipelines in the sense of Continuous Integration (CI), Continuous Delivery (CD), Continuous Deployment, Release Management and Release Orchestration. Or "Application Lifecycle Management" (ALM) - as the total picture is often called. 13 | 14 | The pipeline definition is configurable. The execution configuration of the pipeline definition is decoupled from the pipeline definition, so a single pipeline definition can be executed with different execution configurations. This makes it possible, to just have one pipeline definition and as many execution configurations you want without the need of "cascading project" or "template inheritance" as implemented by other CI tools. 15 | 16 | ### Example configurations 17 | 18 | #### Simple Continuous Integration pipeline 19 | Pipeline definition: 20 | 21 | ```groovy 22 | environment { 23 | mvn: '3.0.5' 24 | git: '1.7.9.5' 25 | env.vars { 26 | EDITOR: 'nano' 27 | } 28 | } 29 | 30 | configuration { 31 | scm(name: latest, type: git) { 32 | url: assertGitUrl 33 | branch: assertString 34 | username: assertString 35 | email: assertEmail 36 | key { 37 | type: assertString 38 | private: assertString 39 | } 40 | } 41 | } 42 | 43 | environment.afterConfigure { 44 | prepare scm { 45 | file('.gitconfig') << ''' 46 | [user] 47 | name = ${scm.latest.username} 48 | email = ${scm.latest.email} 49 | ''' 50 | file('.ssh/id_rsa') << '${scm.latest.key.private}' 51 | } 52 | 53 | verify scm { 54 | run 'git ls-remote --exit-code ${scm.latest.url} ${scm.latest.branch}' 55 | } 56 | } 57 | 58 | stage commit { 59 | description: 'Compile and Test' 60 | run { 61 | env.vars { 62 | SOME_VAR: 'some value' 63 | } 64 | command: git clone ${scm.latest.url} --branch ${scm.latest.branch} 65 | } 66 | run 'mvn clean install' 67 | } 68 | 69 | stage document(dependsOn: commit) { 70 | description: 'Generate documentation' 71 | run 'mvn -P generate-docs' 72 | } 73 | 74 | stage inspect(dependsOn: commit) { 75 | description: 'Inspect the quality' 76 | run 'mvn sonar:sonar' 77 | } 78 | 79 | stage snapshot-release(dependsOn: inspect) { 80 | description: 'Do a snapshot release' 81 | run 'mvn deploy' 82 | } 83 | 84 | announce(type: email) { 85 | to: 'list@mailing.org' 86 | } 87 | ``` 88 | 89 | Required configuration to provide to the pipeline: 90 | 91 | ```groovy 92 | scm(name: latest) { 93 | url: 'git@github.com/myuser/myrepo' 94 | branch: 'develop' 95 | username: 'CI server' 96 | email: 'ci@mydomain.tld' 97 | key.private: '....' 98 | } 99 | ``` 100 | 101 | When executed, the above pipeline would do the following: 102 | 103 | 1. Build an execution model based on provided configuration and the pipeline definition, containing: 104 | * a model that represents the required environment; 105 | * a configuration model based on provided configuration; 106 | * a configuration verification model based the configuration section of the pipeline definition; and 107 | * a Directed Acyclic Graph (DAG) of the stages that looks as following: 108 | * _commit_ 109 | * _document_ 110 | * _inspect_ 111 | * _snapshot-release_ 112 | 113 | 2. Prepare an execution environment containing: 114 | * maven version 3.0.5 on the PATH 115 | * git version 1.7.9.5 on the PATH 116 | 3. Verify that the provided configuration is valid according to the pipeline definition. 117 | 4. Do some additional environment preparation and verification after the configuration is found valid 118 | (e.g. verify that the provided scm configuration is correct by trying to connect). 119 | 5. Start executing the _commit_ stage, as that's the starting point of this graph of stages: 120 | 1. Start stage _document_ 121 | 2. Start stage _inspect_ 122 | 1. Start stage _snapshot-release_ 123 | 124 | If anything fails in this pipeline, it directly exits with a failure status. 125 | The _document_ and _inspect_ stages could be started in parallel. As they are on the same level, if one of them fails, the other will continue to run until it finishes (successfully or not). At that point the pipeline will exit with a failure status. -------------------------------------------------------------------------------- /docs/Vision-Why-a-new-tool-for-Continuous-Delivery.md: -------------------------------------------------------------------------------- 1 | ## What's wrong with existing CI/CD tools 2 | I hear you think "Another one? There are so many existing ones already!?!" and you're completely right. But if you have ever managed multiple projects/jobs in any existing CI/CD tool, you've probably noticed that the existing tools: 3 | 4 | * Force you to copy jobs even though they only differ in some specific configuration properties 5 | * That provide "template"-based job configurations for minimizing job configuration duplication, often only provide this up to a specific point. Still forcing you to manually verify all jobs that extend/use a template that you've updated to see if the change is applied correctly, which mostly is not the case and thereby loosing the benefits of the template-base job configuration 6 | * Except Travis-CI as far as I know, are mostly GUI-based for configuring jobs. Some provide a REST-api, a CLI tool and/or provide scripting functionality for configuring and controlling the jobs from outside the GUI of the tool meaning you have to write your own tools/scripts around your CI tool to make the tool easier to work for your situation 7 | * Provide no way of experimenting: 8 | 9 | * Plugin upgrade? How can you test if a new plugin version still works as the previous one? Upgrade, restart server, start builds and wait for any failing builds...? Sadly this is the reality for existing tools and it's really not uncommon for new plugin versions to be broken! 10 | * Using svn, maven, ant, gradle, git, or any other external tools to get your job work? How do you control those tools? Can you have multiple versions of the same tool? Can you really control the environment your job is executed in/on? As far as I've found noone of the existing tools, that you can run in-house, help with this. Travis-CI does good job of providing jobs with specific versions of tools, but then again, they fully control their environments (there is no in-house version of Travis-CI) 11 | * Have a lot of plugins. Are these plugins really needed? Do they provide some special functionality? Or are they really just some extra configuration options for your jobs, wrappers around external (CLI) tools or visualizing a reporting file from a specific tool 12 | * Provide mostly no change history on job configuration. Configuring jobs can be tricky and often doesn't work as expected the first way around, in existing tools this mean clicking around the GUI (or write tools/scripts around provided API' where available). Something worked a sec ago, it doesn't now, what did you change? No way of telling 13 | 14 | ## Practice what we preach 15 | For our code and infrastructure we've best practices like: 16 | * Configuration-as-Code 17 | * Infrastructure-as-Code 18 | * Version Control 19 | * Keep It Simple Stupid 20 | * Don't Repeat Yourself 21 | * Single Responsibility Principle 22 | 23 | And we focus on applying this practices as good and useful as possible. But it seems like when it comes to CI, all this is thrown overboard. We just accept these CI tools which do not follow the same best practices. While these CI tools automate most of the things that happens with our code and infrastructure til it's on production (and beyond?). Isn't this a bit crooked/awry/weird? 24 | 25 | # This CD tool 26 | So because of all the things above, I want to set things right by developing this new CD tool. 27 | 28 | ## Why write my own 29 | I've looked at many different CI/CD tools and, where available, I've looked at their code, played with their GUI's and inspected their directory structures. And I came to the conclusion that those tools are simply implemented differently in a way it's very hard to make right (in my opinion of course). So I've decided to start with a clean-slate to not have to think about any backwards-compatibility issues of any kind and just go for the best solution possible with the features we want. 30 | I'm writing this tool on the JVM by choice, currently in Java but the JVM provides the possibility to use the best language for the job. Meaning I could be using Groovy in some parts, JRuby in others, Jython, Scala, or any other language available on the JVM. 31 | 32 | ### Focus points 33 | In summary the focus points of this CD server: 34 | * Everything version-controlled: 35 | * Configuration-as-Code 36 | * Infrastructure-as-Code 37 | * Audit-able 38 | * Server runs as daemon without any GUI: 39 | * The daemon provides an API 40 | * A CLI client will be created that uses the provided daemon API 41 | * A Web GUI will be created that uses the provided daemon API 42 | * Without interrupting any of your teams, experiment with existing job configurations: 43 | * Freely 44 | * With new versions of plugins 45 | * With new versions of external tools 46 | * With new configurations of external tools 47 | * Have scriptable job configurations in the domain specific language 48 | * Have plugins that really provide special functionality 49 | * Distributed (in-house and/or in the cloud) 50 | * Lightweight 51 | * Provides hooks to monitor the daemons 52 | * Provides user-type specific ways of working with the tool, eg: 53 | * Developers a way to focus on solving broken builds 54 | * Pipeline maintainers a way to simply create and maintain the pipelines 55 | * Manual QA a way to pull down the version that's ready to manual test 56 | * Manager a way to release a version that came correctly through the pipeline -------------------------------------------------------------------------------- /subprojects/listener/src/test/groovy/cd/pipeline/listen/resources/GitLabWebHookResourceSpec.groovy: -------------------------------------------------------------------------------- 1 | package cd.pipeline.listen.resources 2 | 3 | import cd.pipeline.listen.core.GitTriggerEvent 4 | import com.google.common.eventbus.EventBus 5 | import com.sun.jersey.api.client.Client 6 | import com.sun.jersey.api.client.ClientResponse 7 | import com.sun.jersey.api.client.WebResource 8 | import com.yammer.dropwizard.testing.ResourceTest 9 | import spock.lang.Specification 10 | import spock.lang.Unroll 11 | 12 | import static com.sun.jersey.api.client.ClientResponse.Status.BAD_REQUEST 13 | import static com.sun.jersey.api.client.ClientResponse.Status.NO_CONTENT 14 | import static com.yammer.dropwizard.testing.JsonHelpers.jsonFixture 15 | import static javax.ws.rs.core.MediaType.APPLICATION_JSON_TYPE 16 | 17 | class GitLabWebHookResourceSpec extends Specification { 18 | 19 | def eventBus = Mock(EventBus) 20 | TestResource resource = new TestResource(eventBus) 21 | 22 | @Unroll 23 | def "Valid payload '#payloadFixture' results in git event with gitlab ssh url (#url) and branch (#branch) on queue"() { 24 | given: 25 | def payload = jsonFixture("gitlab/${payloadFixture}") 26 | 27 | when: 28 | def response = requestBuilder() 29 | .type(APPLICATION_JSON_TYPE) 30 | .accept(APPLICATION_JSON_TYPE) 31 | .post(ClientResponse, payload) 32 | 33 | then: 34 | 1 * eventBus.post(_ as GitTriggerEvent) >> { GitTriggerEvent event -> 35 | assert event.url == url 36 | assert event.branch == branch 37 | } 38 | response.type == APPLICATION_JSON_TYPE 39 | response.clientResponseStatus == NO_CONTENT 40 | response.length == -1 41 | 42 | where: 43 | payloadFixture || url | branch 44 | 'samplePayload.json' || 'git@git.domain.com:group/project.git' | 'master' 45 | 'samplePayloadFromBranch.json' || 'git@git.domain.com:group/project.git' | 'somebranch' 46 | } 47 | 48 | def 'Valid payload results in git event on queue'() { 49 | given: 50 | def payload = jsonFixture("gitlab/samplePayload.json") 51 | 52 | when: 53 | def response = requestBuilder() 54 | .type(APPLICATION_JSON_TYPE) 55 | .accept(APPLICATION_JSON_TYPE) 56 | .post(ClientResponse, payload) 57 | 58 | then: 59 | 1 * eventBus.post(_ as GitTriggerEvent) 60 | response.type == APPLICATION_JSON_TYPE 61 | response.clientResponseStatus == NO_CONTENT 62 | response.length == -1 63 | } 64 | 65 | @Unroll 66 | def "POST with '#type' payload results in 204 NO_CONTENT"() { 67 | given: 68 | def payload = jsonFixture("gitlab/${fixture}") 69 | 70 | when: 71 | def response = requestBuilder() 72 | .type(APPLICATION_JSON_TYPE) 73 | .accept(APPLICATION_JSON_TYPE) 74 | .post(ClientResponse, payload) 75 | 76 | then: 77 | response.type == APPLICATION_JSON_TYPE 78 | response.clientResponseStatus == NO_CONTENT 79 | response.length == -1 80 | 81 | where: 82 | type | fixture 83 | 'complete sample' | 'samplePayload.json' 84 | 'minimal sample' | 'minimalPayload.json' 85 | } 86 | 87 | @Unroll 88 | def "POST missing '#missing' in payload results in 400 BAD_REQUEST"() { 89 | given: 90 | def payload = jsonFixture("gitlab/${fixture}") 91 | 92 | when: 93 | def response = requestBuilder() 94 | .type(APPLICATION_JSON_TYPE) 95 | .accept(APPLICATION_JSON_TYPE) 96 | .post(ClientResponse, payload) 97 | 98 | then: 99 | response.type == APPLICATION_JSON_TYPE 100 | response.clientResponseStatus == BAD_REQUEST 101 | 102 | where: 103 | missing | fixture 104 | 'commits' | 'minimalPayloadMissingCommits.json' 105 | 'pusher' | 'minimalPayloadMissingPusher.json' 106 | 'ref' | 'minimalPayloadMissingRef.json' 107 | 'repository.homepage' | 'minimalPayloadMissingRepositoryHomepage.json' 108 | 'repository.name' | 'minimalPayloadMissingRepositoryName.json' 109 | 'repository.url' | 'minimalPayloadMissingRepositoryUrl.json' 110 | } 111 | 112 | def setup() { 113 | resource.setUpJersey() 114 | } 115 | 116 | def cleanup() { 117 | resource.tearDownJersey() 118 | } 119 | 120 | private WebResource.Builder requestBuilder() { 121 | return resource.client().resource('/providers/gitlab').requestBuilder 122 | } 123 | 124 | private class TestResource extends ResourceTest { 125 | 126 | private EventBus bus 127 | 128 | TestResource(EventBus bus) { 129 | this.bus = bus 130 | } 131 | 132 | @Override 133 | protected void setUpResources() throws Exception { 134 | addResource(new GitLabWebHookResource(bus)) 135 | } 136 | 137 | @Override 138 | protected Client client() { 139 | return super.client() 140 | } 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /subprojects/listener/src/test/groovy/cd/pipeline/listen/resources/GitHubWebHookResourceSpec.groovy: -------------------------------------------------------------------------------- 1 | package cd.pipeline.listen.resources 2 | 3 | import cd.pipeline.listen.core.GitTriggerEvent 4 | import com.google.common.eventbus.EventBus 5 | import com.sun.jersey.api.client.Client 6 | import com.sun.jersey.api.client.ClientResponse 7 | import com.sun.jersey.api.client.WebResource 8 | import com.sun.jersey.api.representation.Form 9 | import com.yammer.dropwizard.testing.ResourceTest 10 | import spock.lang.Specification 11 | import spock.lang.Unroll 12 | 13 | import static com.sun.jersey.api.client.ClientResponse.Status.BAD_REQUEST 14 | import static com.sun.jersey.api.client.ClientResponse.Status.NO_CONTENT 15 | import static com.yammer.dropwizard.testing.JsonHelpers.jsonFixture 16 | import static javax.ws.rs.core.MediaType.APPLICATION_FORM_URLENCODED_TYPE 17 | import static javax.ws.rs.core.MediaType.APPLICATION_JSON_TYPE 18 | 19 | class GitHubWebHookResourceSpec extends Specification { 20 | 21 | def eventBus = Mock(EventBus) 22 | TestResource resource = new TestResource(eventBus) 23 | 24 | def 'Valid payload results in git event with github ssh url on queue'() { 25 | given: 26 | def form = new Form() 27 | form.add('payload', jsonFixture("github/samplePayload.json")) 28 | 29 | when: 30 | def response = requestBuilder() 31 | .type(APPLICATION_FORM_URLENCODED_TYPE) 32 | .accept(APPLICATION_JSON_TYPE) 33 | .post(ClientResponse, form) 34 | 35 | then: 36 | 1 * eventBus.post(_ as GitTriggerEvent) >> { GitTriggerEvent event -> 37 | assert event.url == 'git@github.com:/octokitty/testing.git' 38 | assert event.branch == 'master' 39 | } 40 | response.type == APPLICATION_JSON_TYPE 41 | response.clientResponseStatus == NO_CONTENT 42 | response.length == -1 43 | } 44 | 45 | def 'Valid payload results in git event on queue'() { 46 | given: 47 | def form = new Form() 48 | form.add('payload', jsonFixture("github/samplePayload.json")) 49 | 50 | when: 51 | def response = requestBuilder() 52 | .type(APPLICATION_FORM_URLENCODED_TYPE) 53 | .accept(APPLICATION_JSON_TYPE) 54 | .post(ClientResponse, form) 55 | 56 | then: 57 | 1 * eventBus.post(_ as GitTriggerEvent) 58 | response.type == APPLICATION_JSON_TYPE 59 | response.clientResponseStatus == NO_CONTENT 60 | response.length == -1 61 | } 62 | 63 | @Unroll 64 | def "POST with '#type' payload results in 204 NO_CONTENT"() { 65 | given: 66 | def form = new Form() 67 | form.add('payload', jsonFixture("github/${fixture}")) 68 | 69 | when: 70 | def response = requestBuilder() 71 | .type(APPLICATION_FORM_URLENCODED_TYPE) 72 | .accept(APPLICATION_JSON_TYPE) 73 | .post(ClientResponse, form) 74 | 75 | then: 76 | response.type == APPLICATION_JSON_TYPE 77 | response.clientResponseStatus == NO_CONTENT 78 | response.length == -1 79 | 80 | where: 81 | type | fixture 82 | 'complete sample' | 'samplePayload.json' 83 | 'minimal sample' | 'minimalPayload.json' 84 | } 85 | 86 | @Unroll 87 | def "POST missing '#missing' in payload results in 400 BAD_REQUEST"() { 88 | given: 89 | def form = new Form() 90 | form.add('payload', jsonFixture("github/${fixture}")) 91 | 92 | when: 93 | def response = requestBuilder() 94 | .type(APPLICATION_FORM_URLENCODED_TYPE) 95 | .accept(APPLICATION_JSON_TYPE) 96 | .post(ClientResponse, form) 97 | 98 | then: 99 | response.type == APPLICATION_JSON_TYPE 100 | response.clientResponseStatus == BAD_REQUEST 101 | 102 | where: 103 | missing | fixture 104 | 'commits' | 'minimalPayloadMissingCommits.json' 105 | 'head_commit' | 'minimalPayloadMissingHeadCommit.json' 106 | 'pusher' | 'minimalPayloadMissingPusher.json' 107 | 'ref' | 'minimalPayloadMissingRef.json' 108 | 'repository.name' | 'minimalPayloadMissingRepositoryName.json' 109 | 'repository.master_branch' | 'minimalPayloadMissingRepositoryMasterBranch.json' 110 | 'repository.language' | 'minimalPayloadMissingRepositoryLanguage.json' 111 | 'repository.private' | 'minimalPayloadMissingRepositoryPrivateState.json' 112 | 'repository.url' | 'minimalPayloadMissingRepositoryUrl.json' 113 | } 114 | 115 | def setup() { 116 | resource.setUpJersey() 117 | } 118 | 119 | def cleanup() { 120 | resource.tearDownJersey() 121 | } 122 | 123 | private WebResource.Builder requestBuilder() { 124 | return resource.client().resource('/providers/github').requestBuilder 125 | } 126 | 127 | private class TestResource extends ResourceTest { 128 | 129 | private EventBus bus 130 | 131 | TestResource(EventBus bus) { 132 | this.bus = bus 133 | } 134 | 135 | @Override 136 | protected void setUpResources() throws Exception { 137 | addResource(new GitHubWebHookResource(bus)) 138 | } 139 | 140 | @Override 141 | protected Client client() { 142 | return super.client() 143 | } 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ############################################################################## 4 | ## 5 | ## Gradle start up script for UN*X 6 | ## 7 | ############################################################################## 8 | 9 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 10 | DEFAULT_JVM_OPTS="" 11 | 12 | APP_NAME="Gradle" 13 | APP_BASE_NAME=`basename "$0"` 14 | 15 | # Use the maximum available, or set MAX_FD != -1 to use that value. 16 | MAX_FD="maximum" 17 | 18 | warn ( ) { 19 | echo "$*" 20 | } 21 | 22 | die ( ) { 23 | echo 24 | echo "$*" 25 | echo 26 | exit 1 27 | } 28 | 29 | # OS specific support (must be 'true' or 'false'). 30 | cygwin=false 31 | msys=false 32 | darwin=false 33 | case "`uname`" in 34 | CYGWIN* ) 35 | cygwin=true 36 | ;; 37 | Darwin* ) 38 | darwin=true 39 | ;; 40 | MINGW* ) 41 | msys=true 42 | ;; 43 | esac 44 | 45 | # For Cygwin, ensure paths are in UNIX format before anything is touched. 46 | if $cygwin ; then 47 | [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"` 48 | fi 49 | 50 | # Attempt to set APP_HOME 51 | # Resolve links: $0 may be a link 52 | PRG="$0" 53 | # Need this for relative symlinks. 54 | while [ -h "$PRG" ] ; do 55 | ls=`ls -ld "$PRG"` 56 | link=`expr "$ls" : '.*-> \(.*\)$'` 57 | if expr "$link" : '/.*' > /dev/null; then 58 | PRG="$link" 59 | else 60 | PRG=`dirname "$PRG"`"/$link" 61 | fi 62 | done 63 | SAVED="`pwd`" 64 | cd "`dirname \"$PRG\"`/" >&- 65 | APP_HOME="`pwd -P`" 66 | cd "$SAVED" >&- 67 | 68 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 69 | 70 | # Determine the Java command to use to start the JVM. 71 | if [ -n "$JAVA_HOME" ] ; then 72 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 73 | # IBM's JDK on AIX uses strange locations for the executables 74 | JAVACMD="$JAVA_HOME/jre/sh/java" 75 | else 76 | JAVACMD="$JAVA_HOME/bin/java" 77 | fi 78 | if [ ! -x "$JAVACMD" ] ; then 79 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 80 | 81 | Please set the JAVA_HOME variable in your environment to match the 82 | location of your Java installation." 83 | fi 84 | else 85 | JAVACMD="java" 86 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 87 | 88 | Please set the JAVA_HOME variable in your environment to match the 89 | location of your Java installation." 90 | fi 91 | 92 | # Increase the maximum file descriptors if we can. 93 | if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then 94 | MAX_FD_LIMIT=`ulimit -H -n` 95 | if [ $? -eq 0 ] ; then 96 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 97 | MAX_FD="$MAX_FD_LIMIT" 98 | fi 99 | ulimit -n $MAX_FD 100 | if [ $? -ne 0 ] ; then 101 | warn "Could not set maximum file descriptor limit: $MAX_FD" 102 | fi 103 | else 104 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 105 | fi 106 | fi 107 | 108 | # For Darwin, add options to specify how the application appears in the dock 109 | if $darwin; then 110 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 111 | fi 112 | 113 | # For Cygwin, switch paths to Windows format before running java 114 | if $cygwin ; then 115 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 116 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 117 | 118 | # We build the pattern for arguments to be converted via cygpath 119 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 120 | SEP="" 121 | for dir in $ROOTDIRSRAW ; do 122 | ROOTDIRS="$ROOTDIRS$SEP$dir" 123 | SEP="|" 124 | done 125 | OURCYGPATTERN="(^($ROOTDIRS))" 126 | # Add a user-defined pattern to the cygpath arguments 127 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 128 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 129 | fi 130 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 131 | i=0 132 | for arg in "$@" ; do 133 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 134 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 135 | 136 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 137 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 138 | else 139 | eval `echo args$i`="\"$arg\"" 140 | fi 141 | i=$((i+1)) 142 | done 143 | case $i in 144 | (0) set -- ;; 145 | (1) set -- "$args0" ;; 146 | (2) set -- "$args0" "$args1" ;; 147 | (3) set -- "$args0" "$args1" "$args2" ;; 148 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;; 149 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 150 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 151 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 152 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 153 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 154 | esac 155 | fi 156 | 157 | # Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules 158 | function splitJvmOpts() { 159 | JVM_OPTS=("$@") 160 | } 161 | eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS 162 | JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME" 163 | 164 | exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@" 165 | -------------------------------------------------------------------------------- /docs/2013-12-29-Hackaton-UI.asciidoc: -------------------------------------------------------------------------------- 1 | At 29th of December 2013, we're doing a Hackaton on a or some UI(s) for Pipeline. The hackers can decide what type of UIs they create (CLI, GUI, Web-UI, ...) and for what user type the UIs are (dev, test, business, wallboard, ...). 2 | 3 | Some requirements for the UIs: 4 | 5 | - uses API provided by Pipeline 6 | - view-only (for now) 7 | - user-type specific 8 | 9 | The API provided by Pipeline, is a REST API which can be found at https://github.com/pipelinecd/client-api. During the hackaton this API will be extended as needed, though the implementation of the API will (most probably) be mocked at this time. 10 | 11 | == Ideas for UIs 12 | === Developer Web-UI 13 | A developer friendly view on the Pipelines 14 | 15 | ==== Use-cases 16 | 17 | ===== Overview 18 | ------- 19 | *Given* multiple pipelines 20 | *When* I go to the overview page 21 | *Then* I get an overview of all known pipelines 22 | ------- 23 | ------- 24 | *Given* multiple pipelines 25 | *When* I go to the overview page 26 | *Then* I can choose between a list and a grid view of the pipelines 27 | ------- 28 | ------- 29 | *Given* multiple pipelines 30 | *When* I go to the overview page 31 | *Then* I filter to only see the pipelines I'm interested in 32 | ------- 33 | 34 | * Overview of (all accessible) pipelines 35 | ** support filtering 36 | ** have different kind of views, like list, grid, ... 37 | 38 | *UI Ideas* 39 | 40 | List view 41 | ------------ 42 | +------+-------------------------------------------------------------------------------+ 43 | | Logo | {menu, or if browser supports... put menu in browser menu} | 44 | | |----------------------------------------------------------------+--------------+ 45 | +------+ | {filter} | 46 | | +--------------+ 47 | | | 48 | | Name Status | 49 | | +-----------------------------------------------------------------------+ | 50 | | Project XXX Running | 51 | | +-----------------------------------------------------------------------+ | 52 | | Project YYY Failed | 53 | | +-----------------------------------------------------------------------+ | 54 | | Project ... Success | 55 | + +-----------------------------------------------------------------------+ + 56 | | ...... ..... | 57 | | +-----------------------------------------------------------------------+ | 58 | | | 59 | | +-----------------------------------------------------------------------+ | 60 | | | 61 | | | 62 | +----------------+---------------------------------------------------------------------+ 63 | ------------ 64 | 65 | Detailed list view 66 | ------------ 67 | +------+----------------------------------------------------------------+--------------+ 68 | | Logo | {menu, or if browser supports... put menu in browser men|} | {filter} | 69 | | +----------------------------------------------------------------+--------------+ 70 | +------+ | 71 | | | 72 | | | 73 | | Name Current stage Status Host | 74 | | +-----------------------------------------------------------------------+ | 75 | | Project XXX Component Test Running host-01-linux | 76 | | +-----------------------------------------------------------------------+ | 77 | | Project YYY Compile Failed host-02-linux-32 | 78 | | +-----------------------------------------------------------------------+ | 79 | | Project AAA Deploy to Prod Success .... | 80 | | +-----------------------------------------------------------------------+ | 81 | | Project ZZZ Deploy to Prod Need action .... | 82 | | +-----------------------------------------------------------------------+ | 83 | | ..... .... ..... .... | 84 | | +-----------------------------------------------------------------------+ | 85 | | | 86 | | | 87 | +--------------------------------------------------------------------------------------+ 88 | ------------ 89 | 90 | Grid view 91 | ------------ 92 | +------+-------------------------------------------------------------------------------+ 93 | | Logo | {menu, or if browser supports... put menu in browser menu} | 94 | | |------------------------------------------------------+---------+--------------+ 95 | +------+ | {sort} | {filter} | 96 | | +---------+--------------+ 97 | | | 98 | | +-----------------+ +-----------------+ +-----------------+ | 99 | | | Project XXX | | Project YYY | | Project ... | | 100 | | | | | | | | | 101 | | | Running | | Failed | | Success | | 102 | | | | | | | | | 103 | | +-----------------+ +-----------------+ +-----------------+ | 104 | | | 105 | + +-----------------+ +-----------------+ +-----------------+ + 106 | | | Project ... | | Project ... | | Project ... | | 107 | | | | | | | | | 108 | | | .... | | .... | | .... | | 109 | | | | | | | | | 110 | | +-----------------+ +-----------------+ +-----------------+ | 111 | | | 112 | +----------------+---------------------------------------------------------------------+ 113 | ------------ 114 | 115 | ===== Pipeline Specific view 116 | ------- 117 | *Given* a specific pipeline 118 | *When* I select the pipeline 119 | *Then* I get a visualization of the pipeline 120 | *and* I have insight in the history of this pipeline 121 | *and* I have insight in the current state (eg. running, failed, ...) 122 | *and* I have the possibility to see more details on specific parts (stages, tasks, ..) of this pipeline 123 | ------- 124 | * Pipeline specific view 125 | ** show queue? 126 | ** specific pipeline overview picture 127 | ** show details on-request 128 | ** option to show history? 129 | 130 | *UI Idea* 131 | ------------ 132 | +------+-------------------------------------------------------------------------------+ 133 | | Logo | {menu, or if browser supports... put menu in browser menu} | 134 | | |-------------------------------------------------------------------------------+ 135 | +------+ | 136 | | +------+ +-------+ {pipeline workflow view | 137 | | +>| |+-+ +->| |+---+ - zoomable | 138 | | +-----+ | +------+ | +-----+ | +-------+ | - scrollable | 139 | | | +--+ +->| |+-+ +-> - resizable | 140 | | +-----+ + | +-----+ + +-------+ - onclick/onkey show | 141 | | | +------+ | +->| | details in lower | 142 | | +>| |+-+ +-------+ section | 143 | | +------+ } | 144 | | | 145 | +----------------+---------------------------------------------------------------------+ 146 | | | | 147 | | { execution | { details screen | 148 | | specs/stats | - console / log | 149 | | } | - report | 150 | | | - .... | 151 | | | } | 152 | +----------------+---------------------------------------------------------------------+ 153 | ------------ 154 | 155 | 156 | 157 | * Pipeline statistics... 158 | ** For one specific pipeline 159 | ** For all pipelines 160 | 161 | == Technologies ideas 162 | * REST API 163 | ** DropWizard - http://www.dropwizard.io 164 | ** DropWizard with Websockets - https://github.com/mgutz/dropwizard-atmosphere 165 | * WebApp 166 | ** AngularJs - http://angularjs.org 167 | * Monitoring 168 | ** DropWizard Dashboard - https://github.com/kimble/dropwizard-dashboard -------------------------------------------------------------------------------- /docs/Technical-Design-System-design.md: -------------------------------------------------------------------------------- 1 | # Architecture Principles 2 | 3 | - Keep it Simple, Stupid 4 | - Don't Repeat Yourself 5 | - Follow the Unix Philosophy: 6 | 7 | > Write programs that do one thing, and do it well. Write programs that work together. 8 | > -- http://www.linfo.org/unix_philosophy.html 9 | 10 | Each program being a micro-service of the complete picture. 11 | - Each micro-service: 12 | - Has its own git repo 13 | - Has its own versioning/release 14 | - Exposes: 15 | - Metrics 16 | - Logging 17 | - API 18 | - Packaged as a single executable jar 19 | - Installable as you would with any other application, via an OS-dependant package, incl: 20 | - its configuration 21 | - its command script 22 | - its OS dependend "start as daemon" scripts (eg. unix rc.d scripts) 23 | - Micro-service theory presentations: 24 | - From Macro To Micro - Sam Newman 25 | http://www.youtube.com/watch?v=2ofzdPXeQ6o 26 | - Implementing Micro service architectures - Fred George 27 | https://vimeo.com/79866979 28 | - Micro Services: Java, the Unix Way - James Lewis 29 | http://www.infoq.com/presentations/Micro-Services 30 | 31 | ## Todo 32 | - Determine universal dataformat and protocol for the micro-services to communicate, candidates are: 33 | - text streams 34 | - atom/json over http 35 | - messagebus/eventstream 36 | - ...? 37 | 38 | # Design Roadmap 39 | 40 | Namings in this roadmap are work-in-progress, just like the designs 41 | 42 | ## v1 43 | Starting with a single application that does it all in a single process, calling external commands from the `Runner`. 44 | ``` 45 | +---------------------------------------------------+ 46 | | Client | 47 | |---------------------------------------------------| 48 | | +----+ | 49 | | +------------+ uses | | uses +----------+ | 50 | | | Groovy DSL |<-------+Main+------>| Output | | 51 | | +--------+---+ +-+--+ +----------+ | 52 | | | | | 53 | | |uses |uses | 54 | | | | | 55 | | +--------v--------------v---------------------+ | 56 | | | Command-Query API | | 57 | | +---------------------------------------------+ | 58 | | ^ | 59 | | |impl. | 60 | | | | 61 | | +-----------------------+---------------------+ | 62 | | | Runner | | 63 | | +---------------------------------------------+ | 64 | +---------------------------------------------------+ 65 | ``` 66 | (now called `pipe` or `pipe-runner`) 67 | 68 | ## v2 69 | Start moving out the `Runner` to its own application, introducing a `Coordinator` to coordinate the runner processes. 70 | ``` 71 | +----------------------------------------+ 72 | | Client | 73 | |----------------------------------------| 74 | | +----+ | 75 | | +------------+ | | +----------+ | 76 | | | Groovy DSL | |Main| | Output | | 77 | | +------------+ | | +----------+ | 78 | | | | | 79 | | +---------------+----+-------------+ | 80 | | | Command-Query API | | 81 | | +-----------------+----------------+ | 82 | | | | 83 | | +-----------------v----------------+ | 84 | | | Coordinator | | 85 | | +-----------------+----------------+ | 86 | +--------------------|-------------------+ 87 | | 88 | +-----------------v----------------+ 89 | | Runner(s) |-+ 90 | +----------------------------------+ |+ 91 | +-----------------------------------+| 92 | +-----------------------------------+ 93 | ``` 94 | 95 | ## v3 96 | Move out the `Coordinator` to its own application as it proved to work in v2. 97 | ``` 98 | +----------------------------------------+ 99 | | Client | 100 | |----------------------------------------| 101 | | +----+ | 102 | | +------------+ | | +----------+ | 103 | | | Groovy DSL | |Main| | Output | | 104 | | +------------+ | | +----------+ | 105 | | | | | 106 | | +---------------+----+-------------+ | 107 | | | Command-Query API | | 108 | | +-----------------+----------------+ | 109 | | | | 110 | +--------------------|-------------------+ 111 | | 112 | +-----------------v----------------+ 113 | | Coordinator | 114 | +-----------------+----------------+ 115 | | 116 | +-----------------v----------------+ 117 | | Runner(s) |-+ 118 | +----------------------------------+ |+ 119 | +-----------------------------------+| 120 | +-----------------------------------+ 121 | ``` 122 | 123 | ## v.. 124 | ``` 125 | +-----------------------+ 126 | | pipe-poll | 127 | |-----------------------| 128 | | CSV | 129 | | SVN+-+ + | 130 | | | | | 131 | Gitlab webhook | v v | 132 | + | +--+Providers | 133 | | | | | 134 | GitHub webhook | | +-----------+ | 135 | + | +--------------+| pipe-poll | | 136 | | | | | +-----------+ | 137 | +--+ | | +-----------------------+ 138 | | | | 139 | +-----------|--|--|------------------------------------------+ 140 | | | | | pipe-listen | 141 | |-----------|--|--|------------------------------------------| 142 | | | | | | 143 | | v v v | 144 | | +-------------+ | 145 | | | pipe-listen +-------------+Providers | 146 | | +--------+----+ + + + | 147 | | | +---+ | +-----+ | 148 | | | | | v | 149 | | | v | Pipe-poller | 150 | | | GitHub | | 151 | | | v | 152 | | | Gitlab | 153 | | | | 154 | +-------------------|----------------------------------------+ 155 | | 156 | | +-----------------------------------------+ 157 | | | pipe-monitor | 158 | v |-----------------------------------------| 159 | +----------------------------------+ | +---------------+ | 160 | | event-queue / message-bus |<---------+ pipe-monitor +-----+Publishers | 161 | +----------------------------------+ | +---------------+ + + | 162 | ^ ^ ^ ^ | | v | 163 | | | | | | v Graphite | 164 | | | | | | Ganglia | 165 | +----------------|---------+ | | | +-----------------------------------------+ 166 | | pipe-messenger | | | | | 167 | |----------------|---------| | | | +-----------------------------------------+ 168 | | +-------+------+ | | | | | pipe-client-api | +------+ 169 | | |pipe-messenger| | | | | |-----------------------------------------| +-------+| 170 | | +-----+--------+ | | | | | +---------------+ +------------------+ +--------+|+ 171 | | | | | | +------------| event-store +------+ REST <-----+| Web-UI |+ 172 | | + | | | | +---------------+ | <-----++--------+ 173 | | Publishers | | | | | WebSocket/SSE <-----++--------+ 174 | | + + + + | | | +------------------------| <-----+| CLI | 175 | | +--+ | | +-+ | | +-----|-----------------------------------+ +------------------+ +--------+ 176 | | v | | v | | | | pipe-worker | 177 | | Campfire | | Email | | |-----|-----------------------------------| 178 | | | | | | | | | 179 | | v v | | | +--+--------+ | 180 | | Gitlab GitHub | | | |pipe-worker+-------+scm-providers | 181 | +--------------------------+ | | +-+---------+ + + + | 182 | | | | +-+ | +-+ | 183 | | | | v | v | 184 | | | | Git v CSV | 185 | | | | SVN | 186 | | | | | 187 | | +----|------------------------------------+ 188 | | | 189 | | | 190 | | +--|---------------------+ 191 | | | | pipe-runner | 192 | | |--|---------------------| 193 | | | | | 194 | +----+ +> | 195 | | | 196 | +------------------------+ 197 | ``` 198 | 199 | (diagrams simply drawn with http://www.asciiflow.com) -------------------------------------------------------------------------------- /docs/Technical-Design-Braindump.textile: -------------------------------------------------------------------------------- 1 | Workflow-based automated task executor 2 | Example specific usage: Continuous Integration server that is fully Continuous Delivery enabled. 3 | 4 | h2. Focus points 5 | 6 | * Generic purpose 7 | * Workflow-based 8 | * Extensible through plugins 9 | * Version-controlled 10 | ** Configuration-as-Code 11 | ** Infrastructure-as-Code 12 | ** Audit-able 13 | * Distributed (master-slave?) 14 | * Don't re-invent the wheel, reuse/integrate existing tools/libraries, ideas, principles 15 | * Lightweight 16 | * Monitored / Usage Statistics 17 | 18 | h2. Feature ideas 19 | 20 | * workflow/Pipeline support, meaning 21 | ** a pipeline describes the complete delivery process of a project 22 | ** any job with the right configuration can be runned through any pipeline 23 | ** multiple -pipelines- jobs of a project can run in parallel 24 | ** per -pipeline- job run, duplication of steps and files is minimized, meaning 25 | *** data sharing within stages and jobs of a pipeline is encouraged 26 | ** can exist of multiple stages, each stage 27 | *** can run in parallel with other stages in the same pipeline 28 | *** can stop execution of the pipeline if it requires the previous stage to be succesful 29 | *** can exist of multiple jobs, each job 30 | **** can run in parallel with other jobs in the same stage 31 | ** each stage and job has 32 | *** an order of execution 33 | *** an exitCode 34 | * -Workflow/pipeline templates support- Should not be needed as any job can be executed through the pipeline 35 | * Trail-and-error mode, meaning 36 | ** No configuration is used in auto-triggered/scheduled jobs, until that configuration is published to be "production ready" 37 | ** When a job is started it takes a copy of its configuration and uses that during its run, making it undependent of later changes of the job configuration during the job run 38 | ** Configuration changes can be tried and tested manually without affecting any of the auto-triggered/scheduled jobs. Making managing the pipelines as safe as it can be (keep in mind that external resources like nexus don't know of such a mode, so they can still be affected by the jobs runned in trail-and-error mode) 39 | ** Plugin updates can be tested the same way as the configuration changes above 40 | * Configuration-as-code, meaning 41 | ** All configuration are available as files and separated from any dynamically generated files. Making it possible to put the configuration under version control and there audit-able 42 | * Infrastructure-as-code, meaning 43 | ** All tools required for the pipelines are managed through configuration files (by means of Puppet/Chef/..) 44 | ** Multiple versions of tools are supported (but *may* require separated build servers for specific tool versions) 45 | * Distributed, meaning 46 | ** Multiple build servers can be used 47 | ** Light weigth frontend which doesn't need to be on a build-server. Therefore can have a constant good performance 48 | ** Locks & Latches support to discourage jobs from executing in parallel (locks) or to encourage jobs to execute in parallel (latches): 49 | *** Each lock/latch needs to be configurable to set globally or per Hudson instance. Often jobs can't run in parallel on the same hudson instance or destination server (eg. for deployment), but they're fine running in parellel on different hudson instances or destination servers 50 | ** Limited sharing, meaning: 51 | *** Whenever possible, data is shared between tasks, but must be limited, eg: 52 | **** For a Maven task; private maven repo's can be partly shared between tasks, minimizing duplicated networking and disk-write actions 53 | **** For a compilation task; compiled code shared between multiple tasks, minimizing duplicated compilation and disk-write actions (google distributed build filesystem idea) 54 | * Any project can be delivered with this continuous delivery server, as long as the specific build tools are available (java, ruby, groovy, python, maven, ant, bundlr, ...) 55 | * Jobs should never be killed in the middle of their builds: 56 | ** When a node with running jobs on it is taken offline on purpose, the jobs running on it should be completed and not killed 57 | ** When a node goes offline by accident (for whatever reason), the jobs running on that node need to be re-triggered on another node 58 | * Personalized/Private builds, meaning: 59 | ** Grid can be used to execute on a personal/private resource ((d)vcs source) 60 | * Monitored so usage, performance, failure(, etc.?) statistics are gathered and exposed: 61 | ** These should be made available for external use. Maybe messagebus where "backend-frontends" can subscribe one. 62 | 63 | h2. Feature ideas 2 64 | 65 | * A job manager, instead of Locks & Latches, a manager that takes control of a set of jobs/stages/pipelines with the possibility to configure that the jobs need to run in parallel or not 66 | * Eventqueue/Worker setup. Tasks should be executed thru some kind of eventqueue/worker system so tasks can be executed and watched on the background. Even better will be when all tasks are executed by seperated processes, meaning when one process hangs non of the other processes are influenced by it (except for running slower maybe), making it possible to kill the specific problematic process 67 | * Build tasks of a job need to have settings/seperation for: 68 | ** should the task 'always' be executed? (undependent if previous task(s) failed) 69 | ** should task failure/success state influence the state of the job? 70 | (Or have a way to define different type of tasks, eg. init, build, cleanup.. like maven phases. That way states can be tracked per type/phase) 71 | ** Possibility to define what workflow/pipeline should be followed per way that the job is triggered, eg: 72 | *** on manual trigger; just execute the specific job and stop after that 73 | *** on pipeline trigger; follow workflow 74 | *** on schedule trigger?; .... 75 | * Bulk actions, any UI should support doing bulk actions to make maintenance as easy as possible 76 | * SCM plugins must provide an easy solution for defining security details like ssh-keys, eg. via a connection-store(?). The must be no need for manually placing authentication files anywhere then as part of the job config. All authentication details must be kept securily and no passwords should be logged in any log 77 | * Option of mounting the commands and reports as a fuse file system for unix based servers. This again increases the options for easy integration. eg. a /proc or /sys or /dev a-like view on the builds 78 | 79 | h2. Name ideas 80 | 81 | * "Hive" or "colony"; from a bees hive/colony, having a queen-bee (master), ({specific}) worker-bee(s) (slaves), ({specific}) drone(s) (side-tasks, eg. UI, reporting, stats, logging, ...) 82 | ** For background story check: 83 | *** "Beehive":http://en.wikipedia.org/wiki/Beehive 84 | *** "Queen Bee":http://en.wikipedia.org/wiki/Queen_bee 85 | *** "Worker Bee":http://en.wikipedia.org/wiki/Worker_bee and "Laying Worker Bee":http://en.wikipedia.org/wiki/Laying_worker_bee 86 | *** "Drone Bee":http://en.wikipedia.org/wiki/Drone_(bee) and "Drone":http://en.wikipedia.org/wiki/Drone 87 | * "Pipeline" is kinda CI/CD focussed, in the physical world this is called an "Assembly line" 88 | * something related to "Orchestation Platform/Manage" as used in http://vimeo.com/49367413 89 | * Cy / Centurion / Centuri 90 | ** Background info: 91 | *** http://en.wikipedia.org/wiki/CY 92 | *** http://en.wikipedia.org/wiki/Cy_%28Battlestar_Galactica%29 93 | *** http://en.battlestarwiki.org/wiki/Cyrus_%28Cylon%29 94 | *** http://en.battlestarwiki.org/wiki/Centurion_%28TOS%29 95 | *** http://en.wikipedia.org/wiki/Centurion 96 | *** http://en.battlestarwiki.org/wiki/Centuri 97 | *** http://en.wikipedia.org/wiki/Centuri 98 | ** Domain ideas: 99 | *** cy.ci 100 | *** cy.io <- Cy Input/Output 101 | *** centur.io/n 102 | * Continuous (Build) Automation (or Other things/stuff you want to automate) 103 | ** cbaootywta 104 | ** cbaoosywta 105 | ** CiBiAuto 106 | * BuiLd AutomatIoN aka BLAIN 107 | * Build Continuous AutoMation aka BeCAMe 108 | * Automation COntinuous Build aka ACrOBat 109 | * COntinuous Build AuTomation aka aCrOBAT 110 | * Build Automation COntinuous aka BACklOg 111 | * Build Continuous AUtomation aka BeCAUse 112 | * COntinuous Build Automation aka CrOwBAr 113 | * Build AutoMation aka BeAM 114 | * Continuous AutomatioN aka CyAN 115 | * Continuous Your AutomatioN aka CyAN 116 | * Continous Build AuTomation aka CBAT 117 | * Build AutomatiON aka BArON 118 | ** Possible logo: "http://upload.wikimedia.org/wikipedia/commons/thumb/4/40/Ermine_Robe_%28Heraldry%29.svg/265px-Ermine_Robe_%28Heraldry%29.svg.png":http://upload.wikimedia.org/wikipedia/commons/thumb/4/40/Ermine_Robe_%28Heraldry%29.svg/265px-Ermine_Robe_%28Heraldry%29.svg.png 119 | ** Possible Slogans: 120 | *** "Baron has all these butlers working for him. Hudson, Travis, Jenkins." 121 | *** ¨You have butlers.. and then you have the Baron!¨ 122 | ** GreenBaron 123 | * Continuous Integration For All Stakeholders aka CIfas 124 | * Build Automation CONtinuous aka BACON 125 | * FreshCI 126 | * Fresh-CI 127 | * Fresh* 128 | 129 | h1. References for ideas 130 | 131 | * "https://github.com/travis-ci/travis-ci#readme":https://github.com/travis-ci/travis-ci#readme 132 | * "http://www.thoughtworks-studios.com/go-agile-release-management":http://www.thoughtworks-studios.com/go-agile-release-management 133 | * "http://zutubi.com/":http://zutubi.com/ 134 | 135 | h1. Don'ts 136 | 137 | * Don't do project inheritance unless settings can be overridden/inherited per specific property, not a group or section of properties, but per property! Otherwise is still unusable (see Hudson implementation for how it should *not* be implemented 138 | 139 | h1. Others 140 | 141 | * http://www.continuousintegrationtools.com 142 | * http://www.continuousintegrationtutorials.com 143 | * https://github.com/spotify/luigi 144 | * https://github.com/danryan/mastermind 145 | 146 | h2. Travis-CI 147 | 148 | I did some checkup of "Travis-CI":http://www.travis-ci.org, that looks very promising qua infrastructure as they: 149 | 150 | * Infrastructure is split into small focussed parts (worker, hub, build, listener, boxes, core, cookbooks, cli, 151 | * Works with eventqueues and workers 152 | * Makes use of existing services, libraries and tools for everything 153 | * Job configs are version-controlled (placed in the git repo of the job project actually) 154 | 155 | Pros: 156 | * Ideal for integration-test and auto-deploy testing -- Every job run gets a clean virtual-machine (uses snapshotting for quick resets) 157 | 158 | Cons: 159 | * (seems to be) Purely focussed on Github projects 160 | 161 | Note that they are looking into creating a on-demand/in-house solution: "www.travis-ci.com":http://www.travis-ci.com 162 | 163 | h2. CicleCI 164 | 165 | Travis-CI look-a-like 166 | "https://circleci.com":https://circleci.com 167 | 168 | h2. Tddium 169 | 170 | Travis-CI look-a-like 171 | "https://www.tddium.com":https://www.tddium.com 172 | 173 | h2. Semaphore 174 | 175 | Travis-CI look-a-like, specific for ruby 176 | "https://semaphoreapp.com":https://semaphoreapp.com 177 | 178 | h2. Buildkite 179 | 180 | Travis-CI look-a-like 181 | "https://buildkite.com":https://buildkite.com 182 | 183 | h2. Electric Cloud toolset 184 | 185 | After quick look at their site. It looks like the combined tools of electic-cloud come to the same set of features as ThoughtWorks Go and UrbanCode' tools. 186 | "http://www.electric-cloud.com":http://www.electric-cloud.com 187 | 188 | h2. Urban Code toolset 189 | 190 | "http://www.urbancode.com":http://www.urbancode.com 191 | 192 | h2. Rundeck 193 | 194 | RunDeck looks interesting: "http://rundeck.org":http://rundeck.org 195 | It's purpose is system/node orchestration and automation, but it also includes workflow functionality. 196 | Maybe it's usable to look at parts of the functionality? 197 | 198 | h2. Quartz Scheduler library 199 | 200 | "http://www.quartz-scheduler.org":http://www.quartz-scheduler.org looks like the perfect Job execution library to use 201 | 202 | h2. Pulse 203 | 204 | "http://zutubi.com":http://zutubi.com has some nice features like: 205 | * template configuration system 206 | * personal builds 207 | * project dependencies view (a'la hudson up/downstream projects) 208 | 209 | h2. Go 210 | 211 | update 2016-02-28: Go has been open-sourced and is now available as "go.cd":https://www.go.cd 212 | 213 | "http://www.thoughtworks-studios.com/go-agile-release-management":http://www.thoughtworks-studios.com/go-agile-release-management has the pipeline principle in its job configuration naming, but it's still bound to a specific job/project. Templating is available but you shouldn't need templating if a job is only a small bunch of config which you put through the pipeline. 214 | Also, Go Enterprise costs 300 dollar per user per year. And everything that makes configuring jobs a bit easier is only available in the enterprise edition. Incl. Pipeline templates and easy grid usage (not tested this, this is what the version compare shows: "http://www.thoughtworks-studios.com/go-agile-release-management/compare":http://www.thoughtworks-studios.com/go-agile-release-management/compare) 215 | 216 | h2. Continuous Integration in general 217 | 218 | A good write-up of CI in general, CI architectures, etc: 219 | "http://www.aosabook.org/en/integration.html":http://www.aosabook.org/en/integration.html 220 | 221 | Release management side of CI: 222 | "http://niek.bartholomeus.be/2013/05/14/implementing-a-release-management-solution-in-a-traditonal-enterprise/":http://niek.bartholomeus.be/2013/05/14/implementing-a-release-management-solution-in-a-traditonal-enterprise/ 223 | 224 | Managing Build Jobs for Continuous Delivery: 225 | "http://www.infoq.com/articles/Build-Jobs-Continuous-Delivery":http://www.infoq.com/articles/Build-Jobs-Continuous-Delivery 226 | 227 | h1. Implementation resources 228 | 229 | * Using Java as Native Linux Apps – Calling C, Daemonization, Packaging, CLI 230 | "http://java.dzone.com/articles/using-java-native-linux-apps-%E2%80%93":http://java.dzone.com/articles/using-java-native-linux-apps-%E2%80%93 231 | * Good architectural characteristics: 232 | "http://www.infoq.com/news/2012/10/future-monitoring":http://www.infoq.com/news/2012/10/future-monitoring 233 | ** Characteristics like: 234 | *** composable ("well-defined responsibilities, interfaces and protocols") 235 | *** resilient ("resilient to outages within the monitoring architecture") 236 | *** self-service ("doesn't require root access or an Ops member to deploy") 237 | *** automated ("it's capable of being automated") 238 | *** correlative ("implicitly model relationships between services") 239 | *** craftsmanship ("it's a pleasure to use") 240 | ** Possible components (based on a monitoring system): 241 | *** sensors "are stateless agents that gather and emit metrics to a log stream, over HTTP as JSON, or directly to the metrics store" 242 | *** aggregators "are responsible for transformation, aggregation, or possibly simply relaying of metrics" 243 | *** state engine "tracks changes within the event stream, ideally it can ascertain faults according to seasonality and forecasting" 244 | *** storage engines "should support transformative functions and aggregations, ideally should be capable of near-realtime metrics retrieval and output in standard formats such as JSON, XML or SVG" 245 | *** scheduler "provides an interface for managing on-call and escalation calendars" 246 | *** notifiers "are responsible for composing alert messages using data provided by the state engine and tracking their state for escalation purposes" 247 | *** visualizers "consist of dashboards and other user interfaces that consume metrics and alerts from the system" 248 | * Directed Acyclic Graph implementations: 249 | ** "http://plexus.codehaus.org/plexus-utils/apidocs/index.html":http://plexus.codehaus.org/plexus-utils/apidocs/index.html 250 | (as used in Maven) 251 | ** "http://jgrapht.org":http://jgrapht.org 252 | * DSLs in Jenkins: 253 | ** "https://wiki.jenkins-ci.org/display/JENKINS/Build+Flow+Plugin":https://wiki.jenkins-ci.org/display/JENKINS/Build+Flow+Plugin 254 | ** "https://wiki.jenkins-ci.org/display/JENKINS/Job+DSL+Plugin":https://wiki.jenkins-ci.org/display/JENKINS/Job+DSL+Plugin 255 | * Server troubleshooting, the first 5 minutes 256 | ** "http://devo.ps/blog/2013/03/06/troubleshooting-5minutes-on-a-yet-unknown-box.html":http://devo.ps/blog/2013/03/06/troubleshooting-5minutes-on-a-yet-unknown-box.html 257 | * look at "gradle":http://www.gradle.org for implementation ideas 258 | * look at "Tesla":http://tesla.io/tesla/index.html for implementation ideas 259 | 260 | h1. Maintaining an open-source project 261 | * "http://coding.smashingmagazine.com/2013/01/03/starting-open-source-project/":http://coding.smashingmagazine.com/2013/01/03/starting-open-source-project/ 262 | 263 | h1. Slogan explained 264 | 265 | bq. Make the Impossible Possible 266 | 267 | Continuous Integration, Delivery, no way, that will never work... 268 | 269 | bq. Make the Possible Simple 270 | 271 | Sure it will work, just make it simpeler 272 | 273 | bq. Make the Simple Go Away 274 | 275 | Automate 276 | --------------------------------------------------------------------------------