├── .circleci └── config.yml ├── README.md ├── module-a └── .circleci │ └── config.yml ├── module-b └── .circleci │ └── config.yml └── module-c └── .circleci └── config.yml /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | # Config for the setup workflow and common resources (jobs, commands) for main workflows 2 | # `common-` prefix is added for common resources (to avoid conflicts with module resources) 3 | 4 | version: 2.1 5 | 6 | setup: << pipeline.parameters.run-setup >> 7 | 8 | # All pipeline parameters need to be defined equally both for the setup workflow and main workflows 9 | # These parameters will be passed to both of them 10 | parameters: 11 | run-setup: 12 | description: Whether it is a setup workflow or a continuation 13 | type: boolean 14 | default: true 15 | force-all: 16 | description: Emergency valve - forcibly build all the modules 17 | type: boolean 18 | default: false 19 | custom-parameter: 20 | description: Some string to pass 21 | type: string 22 | default: "" 23 | 24 | # Custom commands aimed for the main workflows and jobs 25 | commands: 26 | common-say-hello: 27 | parameters: 28 | to: 29 | description: To whom you say hello 30 | type: string 31 | default: you anonymous 32 | steps: 33 | - run: echo 'Greetings to << parameters.to >> from the shared command!' 34 | - run: echo 'The value of `custom-parameter` was "<< pipeline.parameters.custom-parameter >>"' 35 | 36 | # Common jobs for the main workflows 37 | jobs: 38 | common-pre: 39 | docker: 40 | - image: alpine 41 | steps: 42 | - common-say-hello 43 | - run: echo 'Jobs with `common-` prefix are intended to be shared among modules' 44 | 45 | workflows: 46 | # The setup workflow 47 | setup-workflow: 48 | when: << pipeline.parameters.run-setup >> 49 | jobs: 50 | - config-splitting/setup-dynamic-config: 51 | force-all: << pipeline.parameters.force-all >> 52 | base-revision: main 53 | # If A is modified, the job for A will run 54 | # Similarly if B is modified, the job for B will run 55 | # If C is modified, then the jobs for both C and B (as a dependency) will run 56 | modules: | 57 | module-a 58 | module-b 59 | module-c module-b 60 | 61 | orbs: 62 | # An "embedded" orb to facilitate config splitting 63 | config-splitting: 64 | # Dependencies 65 | orbs: 66 | continuation: circleci/continuation@0.1.2 67 | # Commands for the setup workflow 68 | commands: 69 | list-changed-modules: 70 | parameters: 71 | modules: 72 | description: | 73 | Directories which should be built upon changes. 74 | Each row represents a space-separated list of the root directories for modules, each of which must has own `.circleci/config.yml`. 75 | The first item of the list will be tested for changes, and will be added to the filtered list of modules if there are any changes. 76 | The subsequent items, if there are any, will also be added to the filtered list of modules if there are any changes in the directory specified as the first item. 77 | 78 | CAVEAT: Directory names having white spaces cannot be specified. 79 | type: string 80 | modules-filtered: 81 | description: Path to the file where the filtered list of modules is generated 82 | type: string 83 | default: /tmp/modules-filtered.txt 84 | base-revision: 85 | description: Revision to compare with the current HEAD 86 | type: string 87 | default: main 88 | force-all: 89 | description: Emergency valve - forcibly build all the modules 90 | type: boolean 91 | default: false 92 | steps: 93 | - run: 94 | name: Generate the list of modules having changes 95 | command: | 96 | # Add each module to `modules-filtered` if 1) `force-all` is set to `true`, 2) there is a diff against `base-revision`, 3) there is no `HEAD~1` (i.e., this is the very first commit for the repo) OR 4) there is a diff against the previous commit 97 | cat \<< EOD | sed -e '/^$/d' | while read row; do module="$(echo "$row" | awk '{ print $1 }')"; if [ << parameters.force-all >> == 'true' ] || [ $(git diff --name-only << parameters.base-revision >> "$module" | wc -l) -gt 0 ] || (! git rev-parse --verify HEAD~1) || [ $(git diff --name-only HEAD~1 "$module" | wc -l) -gt 0 ]; then echo "$row" | sed -e 's/ /\n/g' >> << parameters.modules-filtered >>; fi; done 98 | << parameters.modules >> 99 | EOD 100 | 101 | merge-modular-configs: 102 | parameters: 103 | modules: 104 | description: Path to the file for the list of the modules to build 105 | type: string 106 | default: /tmp/modules-filtered.txt 107 | shared-config: 108 | description: Path to the config providing shared resources (such as prerequisite jobs and common commands) 109 | type: string 110 | default: .circleci/config.yml 111 | continue-config: 112 | description: Path to the internally-used config for continuation 113 | type: string 114 | default: .circleci/continue-config.yml 115 | steps: 116 | - run: 117 | name: Merge configs 118 | command: | 119 | # If `modules` is unavailable, stop this job without continuation 120 | if [ ! -f "<< parameters.modules >>" ] || [ ! -s "<< parameters.modules >>" ] 121 | then 122 | echo 'Nothing to merge. Halting the job.' 123 | circleci-agent step halt 124 | exit 125 | fi 126 | 127 | # Convert a list of dirs to a list of config.yml 128 | sed -i -e 's/$/\/.circleci\/config.yml/g' "<< parameters.modules >>" 129 | 130 | # If `shared-config` exists, append it at the end of `modules` 131 | if [ -f << parameters.shared-config >> ] 132 | then 133 | echo "<< parameters.shared-config >>" >> "<< parameters.modules >>" 134 | fi 135 | 136 | xargs -a "<< parameters.modules >>" yq -y -s 'reduce .[] as $item ({}; . * $item)' | tee "<< parameters.continue-config >>" 137 | 138 | jobs: 139 | # The job for the setup workflow 140 | setup-dynamic-config: 141 | parameters: 142 | modules: 143 | description: Directories which should be tested for changes; one directory per line. Each directory must have `.circleci/config.yml`. 144 | type: string 145 | base-revision: 146 | description: Revision to compare with the current HEAD 147 | type: string 148 | default: main 149 | force-all: 150 | description: Emergency valve - forcibly build all the modules 151 | type: boolean 152 | default: false 153 | modules-filtered: 154 | description: Path to the file where the filtered list of modules is generated 155 | type: string 156 | default: /tmp/modules-filtered.txt 157 | shared-config: 158 | description: Path to the config providing shared resources (such as prerequisite jobs and common commands) 159 | type: string 160 | default: .circleci/config.yml 161 | continue-config: 162 | description: Path to the internally-used config for continuation 163 | type: string 164 | default: .circleci/continue-config.yml 165 | docker: 166 | - image: cimg/python:3.9 167 | steps: 168 | - checkout 169 | - run: 170 | name: Install yq 171 | command: pip install yq 172 | - list-changed-modules: 173 | modules: << parameters.modules >> 174 | modules-filtered: << parameters.modules-filtered >> 175 | base-revision: << parameters.base-revision >> 176 | force-all: << parameters.force-all >> 177 | - merge-modular-configs: 178 | modules: << parameters.modules-filtered >> 179 | shared-config: << parameters.shared-config >> 180 | continue-config: << parameters.continue-config >> 181 | - continuation/continue: 182 | configuration_path: << parameters.continue-config >> 183 | parameters: '{"run-setup":false}' 184 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Path filtering + config splitting on CircleCI 2 | 3 | This repository demonstrates an advanced use case of setup workflow feature on CircleCI. For instance, it implements both path filtering and config splitting. 4 | 5 | ## Files 6 | 7 | * `.circleci/config.yml` implements both 1) the setup workflow, and 2) common resources (i.e., jobs and commands) for main workflows/jobs. 8 | * `module-a/.circleci/config.yml`, `module-b/.circleci/config.yml`, and `module-c/.circleci/config.yml` implement independent modular configs for module A, B, and C, respectively. 9 | 10 | ## How does it work? 11 | 12 | 1. Upon the initial trigger, CircleCI triggers the setup job `setup-dynamic-config` defined in `.circleci/config.yml`. 13 | 2. Given a list of directories, detect which subdirectories (herein modules) have changes. (cf. `list-changed-modules`) 14 | 3. Fetch `path-to-module/.circleci/config.yml` for each module to build, and merge all the fetched `config.yml` (along with the config defining common resources, i.e., `.circleci/config.yml`) using `yq`. (cf. `merge-modular-configs`) 15 | 4. Trigger execution of the merged config. 16 | -------------------------------------------------------------------------------- /module-a/.circleci/config.yml: -------------------------------------------------------------------------------- 1 | # Config dedicated for module A 2 | # All the resources must be prefixed with `module-a-` to avoid conflicts 3 | 4 | version: 2.1 5 | 6 | orbs: 7 | # Note: We are using the latest version of hello-build, whereas module B uses an older one 8 | module-a-hello-build: circleci/hello-build@0.0.14 9 | 10 | jobs: 11 | module-a-build: 12 | docker: 13 | - image: alpine 14 | steps: 15 | - common-say-hello: 16 | to: A 17 | - run: echo 'Hello world from module A! :3' 18 | 19 | workflows: 20 | module-a-workflow: 21 | jobs: 22 | - common-pre 23 | - module-a-build: 24 | requires: 25 | - common-pre 26 | - module-a-hello-build/hello-build: 27 | requires: 28 | - common-pre 29 | -------------------------------------------------------------------------------- /module-b/.circleci/config.yml: -------------------------------------------------------------------------------- 1 | # Config dedicated for module B 2 | # All the resources must be prefixed with `module-b-` to avoid conflicts 3 | 4 | version: 2.1 5 | 6 | orbs: 7 | # Note: We are using an older version of hello-build, whereas module A uses the latest 8 | module-b-hello-build: circleci/hello-build@0.0.13 9 | 10 | jobs: 11 | module-b-build: 12 | docker: 13 | - image: alpine 14 | steps: 15 | - common-say-hello: 16 | to: B 17 | - run: echo 'Hello, from module B! :3' 18 | 19 | module-b-postrun: 20 | docker: 21 | - image: alpine 22 | steps: 23 | - run: echo 'Another job for B! :3' 24 | 25 | workflows: 26 | module-b-workflow-x: 27 | jobs: 28 | - common-pre 29 | - module-b-build: 30 | requires: 31 | - common-pre 32 | - module-b-postrun: 33 | requires: 34 | - module-b-build 35 | module-b-workflow-y: 36 | jobs: 37 | - module-b-hello-build/hello-build 38 | -------------------------------------------------------------------------------- /module-c/.circleci/config.yml: -------------------------------------------------------------------------------- 1 | # Config dedicated for module C 2 | # All the resources must be prefixed with `module-c-` to avoid conflicts 3 | 4 | version: 2.1 5 | 6 | orbs: 7 | module-c-hello-build: circleci/hello-build@0.0.14 8 | 9 | jobs: 10 | module-c-build: 11 | docker: 12 | - image: alpine 13 | steps: 14 | - common-say-hello: 15 | to: C 16 | - run: echo 'Hello world from module C! :3' 17 | 18 | workflows: 19 | module-c-workflow: 20 | jobs: 21 | - common-pre 22 | - module-c-build: 23 | requires: 24 | - common-pre 25 | - module-c-hello-build/hello-build: 26 | requires: 27 | - common-pre 28 | --------------------------------------------------------------------------------