├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── build.go ├── build_example_test.go ├── build_package_test.go ├── cached_command.go ├── cmd └── command.go ├── command_runner.go ├── constants.go ├── examples ├── Makefile ├── cmd_add_user.go ├── cmd_as_user.go ├── cmd_bool.go ├── cmd_download.go ├── cmd_extract_file.go ├── cmd_file.go ├── cmd_fileutils.go ├── cmd_shell.go ├── cmd_ubuntu.go ├── cmd_wait.go ├── main.go ├── tpl_cronjob.go ├── tpl_docker.go ├── tpl_elasticsearch.go ├── tpl_firewall.go ├── tpl_golang.go ├── tpl_haproxy.go ├── tpl_jenkins.go ├── tpl_nginx.go ├── tpl_openvpn.go ├── tpl_postgis.go ├── tpl_postgres.go ├── tpl_rabbitmq.go ├── tpl_redis.go ├── tpl_ruby.go ├── tpl_syslogng.go ├── tpl_system.go ├── urknall-hello-example │ ├── cmd_shell.go │ └── main.go └── urknall-rack-example │ ├── Makefile │ ├── app.go │ ├── cmd_add_user.go │ ├── cmd_as_user.go │ ├── cmd_bool.go │ ├── cmd_download.go │ ├── cmd_extract.go │ ├── cmd_fileutils.go │ ├── cmd_shell.go │ ├── cmd_ubuntu.go │ ├── cmd_write_file.go │ ├── main.go │ └── ruby.go ├── extract_file_test.go ├── integration_test.go ├── logger_example_test.go ├── logging.go ├── package.go ├── package_impl.go ├── package_impl_test.go ├── package_test.go ├── pubsub ├── pubsub.go └── stdout_logger.go ├── runlist_test.go ├── string_command.go ├── target.go ├── target ├── command.go ├── local.go ├── ssh.go └── ssh_test.go ├── task.go ├── task_test.go ├── template_func.go ├── urknall.go ├── urknall ├── .gitattributes ├── Makefile ├── base.go ├── github.go ├── init.go ├── main.go ├── router.go ├── templates_add.go └── templates_list.go ├── utils.go ├── utils ├── templates.go ├── templates_test.go ├── version_test.go └── versions.go ├── utils_test.go └── validation.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | 24 | # Ignore compiled config files 25 | assets/assets.go 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: default build test vet 2 | 3 | default: build test vet 4 | 5 | build: 6 | @go get ./... 7 | 8 | test: 9 | @go test $(PACKAGES) 10 | 11 | vet: 12 | @go vet $(PACKAGES) 13 | 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Urknall - opinionated provisioning for clever developers 2 | 3 | [![Documentation](http://img.shields.io/badge/gh--pages-documentation-blue.svg)](http://urknall.dynport.de) 4 | [![GoDoc](https://godoc.org/github.com/dynport/urknall?status.svg)](https://godoc.org/github.com/dynport/urknall) 5 | 6 | [Urknall](http://urknall.dynport.de) is the basic building block for creating 7 | go based tools for the administration of complex infrastructure. It provides 8 | the mechanisms to access resources and keep a cache of executed tasks. 9 | Description of tasks is done using a mix of explicit shell commands and 10 | abstractions for more complex actions being pragmatic, but readable. 11 | 12 | * _Automate provisioning_: Urknall provides automated 13 | provisioning, so that resources can be set up reproducibly and easily. 14 | Thereby Urknall helps with scaling infrastructure to the required size. 15 | * _Agentless tool that only relies on common UNIX tools and provides decent 16 | caching_: As Urknall works agentless on most UNIX based systems, adding new 17 | resources to the infrastructure is effortless. The caching keeps track of 18 | performed tasks and makes repeated provisioning with thoughtful changes 19 | possible. 20 | * _Template mechanism that helps at the start, but get’s out of the way later 21 | on_: Urknall provides some basic templates, but lets users modify those or 22 | add new ones to solve the specific problem at hand. 23 | 24 | urknall is developed and maintained by [Dynport GmbH](http://www.dynport.de), 25 | the company behind the translation management software 26 | [PhraseApp](https://phraseapp.com). 27 | 28 | -------------------------------------------------------------------------------- /build.go: -------------------------------------------------------------------------------- 1 | package urknall 2 | 3 | import ( 4 | "archive/tar" 5 | "bufio" 6 | "bytes" 7 | "compress/gzip" 8 | "encoding/base64" 9 | "encoding/json" 10 | "fmt" 11 | "io" 12 | "io/ioutil" 13 | "os" 14 | "path/filepath" 15 | "strings" 16 | "sync" 17 | "text/template" 18 | "time" 19 | 20 | "github.com/dynport/dgtk/confirm" 21 | "github.com/dynport/gocli" 22 | "github.com/dynport/urknall/pubsub" 23 | "github.com/dynport/urknall/target" 24 | ) 25 | 26 | // A shortcut creating and running a build from the given target and template. 27 | func Run(target Target, tpl Template, opts ...func(*Build)) (e error) { 28 | b := &Build{Target: target, Template: tpl} 29 | for _, o := range opts { 30 | o(b) 31 | } 32 | return b.Run() 33 | } 34 | 35 | // A shortcut creating and runnign a build from the given target and template 36 | // with the DryRun flag set to true. This is quite helpful to actually see 37 | // which commands would be exeucted in the current setting, without actually 38 | // doing anything. 39 | func DryRun(target Target, tpl Template) (e error) { 40 | return (&Build{Target: target, Template: tpl}).DryRun() 41 | } 42 | 43 | // A build is the glue between a target and template. 44 | type Build struct { 45 | Target // Where to run the build. 46 | Template // What to actually build. 47 | Env []string // Environment variables in the form `KEY=VALUE`. 48 | Confirm func(actions ...*confirm.Action) error 49 | 50 | maxLength int // length of the longest key to be executed 51 | } 52 | 53 | // This will render the build's template into a package and run all its tasks. 54 | func (b *Build) Run() error { 55 | i, err := renderTemplate(b.Template) 56 | if err != nil { 57 | return err 58 | } 59 | m, err := readState(b.Target) 60 | if err != nil { 61 | return err 62 | } 63 | actions := confirm.Actions{} 64 | 65 | for _, t := range i.tasks { 66 | ex := []string{} 67 | if s, ok := m[t.name]; ok { 68 | ex = s.runSHAs 69 | } 70 | 71 | diff := []string{} 72 | checksums := []string{} 73 | broken := false 74 | for i, c := range t.commands { 75 | cs := c.Checksum() 76 | checksums = append(checksums, "/var/lib/urknall/"+t.name+"/"+cs+".done") 77 | if broken || len(ex) <= i || ex[i] != cs { 78 | diff = append(diff, cs) 79 | broken = true 80 | 81 | var pl []byte 82 | _, cmd, ok, err := extractWriteFile(c.command.Shell()) 83 | if err == nil && ok { 84 | pl = []byte(cmd) 85 | } 86 | if len(t.name) > b.maxLength { 87 | b.maxLength = len(t.name) 88 | } 89 | actions.Create(t.name+" "+c.LogMsg(), pl, b.commandAction(t.name, checksums, c)) 90 | } 91 | } 92 | } 93 | 94 | if b.Confirm != nil { 95 | if err := b.Confirm(actions...); err != nil { 96 | return err 97 | } 98 | } else { 99 | for _, a := range actions { 100 | if err := a.Call(); err != nil { 101 | return err 102 | } 103 | } 104 | } 105 | return nil 106 | } 107 | 108 | func (b *Build) DryRun() error { 109 | pkg, e := b.prepareBuild() 110 | if e != nil { 111 | return e 112 | } 113 | 114 | for _, task := range pkg.tasks { 115 | for _, command := range task.commands { 116 | m := message(pubsub.MessageTasksProvisionTask, b.hostname(), task.name) 117 | m.TaskChecksum = command.Checksum() 118 | m.Message = command.LogMsg() 119 | 120 | switch { 121 | case command.cached: 122 | m.ExecStatus = pubsub.StatusCached 123 | m.Publish("finished") 124 | default: 125 | m.ExecStatus = pubsub.StatusExecStart 126 | m.Publish("executed") 127 | } 128 | } 129 | } 130 | return nil 131 | } 132 | 133 | func (build *Build) prepareBuild() (*packageImpl, error) { 134 | pkg, e := renderTemplate(build.Template) 135 | if e != nil { 136 | return nil, e 137 | } 138 | 139 | if e = build.prepareTarget(); e != nil { 140 | return nil, e 141 | } 142 | 143 | ct, e := build.buildChecksumTree() 144 | if e != nil { 145 | return nil, fmt.Errorf("error building checksum tree: %s", e.Error()) 146 | } 147 | 148 | return pkg, build.prepareTasks(ct, pkg.tasks...) 149 | } 150 | 151 | func (build *Build) prepareTarget() error { 152 | if build.User() == "" { 153 | return fmt.Errorf("User not set") 154 | } 155 | rawCmd := fmt.Sprintf(`{ grep "^%s:" /etc/group | grep %s; } && [ -d %[3]s ] && [ -f %[3]s/.v2 ]`, 156 | ukGROUP, build.User(), ukCACHEDIR) 157 | cmd, e := build.prepareInternalCommand(rawCmd) 158 | if e != nil { 159 | return e 160 | } 161 | if e := cmd.Run(); e != nil { 162 | // If user is missing the group, create group (if necessary), add user and restart ssh connection. 163 | cmds := []string{ 164 | fmt.Sprintf(`{ grep -e '^%[1]s:' /etc/group > /dev/null || { groupadd %[1]s; }; }`, ukGROUP), 165 | fmt.Sprintf(`{ [ -d %[1]s ] || { mkdir -p -m 2775 %[1]s && chgrp %[2]s %[1]s; }; }`, ukCACHEDIR, ukGROUP), 166 | fmt.Sprintf("usermod -a -G %s %s", ukGROUP, build.User()), 167 | fmt.Sprintf(`[ -f %[1]s/.v2 ] || { export DATE=$(date "+%%Y%%m%%d_%%H%%M%%S") && ls %[1]s | while read dir; do ls -t %[1]s/$dir/*.done | tac > %[1]s/$dir/$DATE.run; done && touch %[1]s/.v2; } `, ukCACHEDIR), 168 | } 169 | 170 | cmd, e = build.prepareInternalCommand(strings.Join(cmds, " && ")) 171 | if e != nil { 172 | return e 173 | } 174 | out := &bytes.Buffer{} 175 | err := &bytes.Buffer{} 176 | cmd.SetStderr(err) 177 | cmd.SetStdout(out) 178 | if e := cmd.Run(); e != nil { 179 | return fmt.Errorf("failed to initiate user %q for provisioning: %s, out=%q err=%q", build.User(), e, out.String(), err.String()) 180 | } 181 | return build.Reset() 182 | } 183 | return nil 184 | } 185 | 186 | func (build *Build) prepareTasks(ct checksumTree, tasks ...*task) error { 187 | for _, task := range tasks { 188 | if err := build.prepareTask(task, ct); err != nil { 189 | return err 190 | } 191 | } 192 | return nil 193 | } 194 | 195 | func (build *Build) prepareTask(tsk *task, ct checksumTree) (e error) { 196 | cacheKey := tsk.name 197 | if cacheKey == "" { 198 | return fmt.Errorf("CacheKey must not be empty") 199 | } 200 | checksumDir := fmt.Sprintf(ukCACHEDIR+"/%s", tsk.name) 201 | 202 | var found bool 203 | var checksumList []string 204 | 205 | if checksumList, found = ct[cacheKey]; !found { 206 | // Create checksum dir and set group bit (all new files will inherit the directory's group). This allows for 207 | // different users (being part of that group) to create, modify and delete the contained checksum and log files. 208 | createChecksumDirCmd := fmt.Sprintf("mkdir -m2775 -p %s", checksumDir) 209 | 210 | cmd, e := build.prepareInternalCommand(createChecksumDirCmd) 211 | if e != nil { 212 | return e 213 | } 214 | err := &bytes.Buffer{} 215 | 216 | cmd.SetStderr(err) 217 | 218 | if e := cmd.Run(); e != nil { 219 | return fmt.Errorf(err.String() + ": " + e.Error()) 220 | } 221 | } 222 | 223 | // find commands that need not be executed 224 | for i, cmd := range tsk.commands { 225 | checksum, e := commandChecksum(cmd.command) 226 | if e != nil { 227 | return e 228 | } 229 | 230 | switch { 231 | case len(checksumList) <= i || checksum != checksumList[i]: 232 | return nil 233 | default: 234 | cmd.cached = true 235 | } 236 | } 237 | 238 | return nil 239 | } 240 | 241 | func (build *Build) buildTask(tsk *task) (e error) { 242 | checksumDir := fmt.Sprintf(ukCACHEDIR+"/%s", tsk.name) 243 | 244 | tsk.started = time.Now() 245 | 246 | for _, cmd := range tsk.commands { 247 | checksum := cmd.Checksum() 248 | 249 | m := message(pubsub.MessageTasksProvisionTask, build.hostname(), tsk.name) 250 | m.TaskChecksum = checksum 251 | m.Message = cmd.LogMsg() 252 | 253 | var cmdErr error 254 | 255 | switch { 256 | case cmd.cached: 257 | m.ExecStatus = pubsub.StatusCached 258 | default: 259 | m.ExecStatus = pubsub.StatusExecStart 260 | m.Publish("started") 261 | 262 | r := &commandRunner{ 263 | build: build, 264 | command: cmd.command, 265 | dir: checksumDir, 266 | taskName: tsk.name, 267 | } 268 | cmdErr = r.run() 269 | 270 | m.Error = cmdErr 271 | m.ExecStatus = pubsub.StatusExecFinished 272 | } 273 | m.Publish("finished") 274 | err := build.addCmdToTaskLog(tsk, checksumDir, checksum, cmdErr) 275 | switch { 276 | case cmdErr != nil: 277 | return cmdErr 278 | case err != nil: 279 | return err 280 | } 281 | } 282 | 283 | return nil 284 | } 285 | 286 | // addCmdToTaskLog will manage the log of run commands in a file. This file gets append the path to a file 287 | // for each command, that contains the executed script. The filename contains either ".done" or ".failed" as 288 | // suffix, depending on the err given (nil or not). 289 | func (build *Build) addCmdToTaskLog(tsk *task, checksumDir, checksum string, err error) (e error) { 290 | prefix := checksumDir + "/" + checksum 291 | sourceFile := prefix + ".sh" 292 | targetFile := prefix + ".done" 293 | if err != nil { 294 | logError(err) 295 | targetFile = prefix + ".failed" 296 | } 297 | rawCmd := fmt.Sprintf("{ [ -f %[1]s ] || mv %[2]s %[1]s; } && echo %[1]s >> %[3]s/%[4]s.run", 298 | targetFile, sourceFile, checksumDir, tsk.started.Format("20060102_150405")) 299 | c, e := build.prepareInternalCommand(rawCmd) 300 | if e != nil { 301 | return e 302 | } 303 | 304 | return c.Run() 305 | } 306 | 307 | type checksumTree map[string][]string 308 | 309 | func (build *Build) buildChecksumTree() (ct checksumTree, e error) { 310 | ct = checksumTree{} 311 | 312 | rawCmd := fmt.Sprintf( 313 | `[ -d %[1]s ] && { ls %[1]s | while read dir; do ls -t %[1]s/$dir/*.run | head -n1 | xargs cat; done; }`, 314 | ukCACHEDIR) 315 | cmd, e := build.prepareInternalCommand(rawCmd) 316 | if e != nil { 317 | return nil, e 318 | } 319 | out := &bytes.Buffer{} 320 | err := &bytes.Buffer{} 321 | 322 | cmd.SetStdout(out) 323 | cmd.SetStderr(err) 324 | 325 | if e := cmd.Run(); e != nil { 326 | return nil, fmt.Errorf("%s: out=%s err=%s", e.Error(), out.String(), err.String()) 327 | } 328 | 329 | for _, line := range strings.Split(out.String(), "\n") { 330 | line = strings.TrimSpace(line) 331 | 332 | if line == "" || !strings.HasSuffix(line, ".done") { 333 | continue 334 | } 335 | 336 | pkgname := filepath.Dir(strings.TrimPrefix(line, ukCACHEDIR+"/")) 337 | checksum := strings.TrimSuffix(filepath.Base(line), ".done") 338 | if len(checksum) != 64 { 339 | return nil, fmt.Errorf("invalid checksum %q found for package %q", checksum, pkgname) 340 | } 341 | ct[pkgname] = append(ct[pkgname], checksum) 342 | } 343 | 344 | return ct, nil 345 | } 346 | 347 | func (build *Build) prepareCommand(rawCmd string) (target.ExecCommand, error) { 348 | var sudo string 349 | if build.User() != "root" { 350 | sudo = "sudo " 351 | } 352 | return build.Command(sudo + rawCmd) 353 | } 354 | 355 | func (build *Build) prepareInternalCommand(rawCmd string) (target.ExecCommand, error) { 356 | rawCmd = fmt.Sprintf("sh -x -e <<\"EOC\"\n%s\nEOC\n", rawCmd) 357 | return build.prepareCommand(rawCmd) 358 | } 359 | 360 | func (build *Build) hostname() string { 361 | if s, ok := build.Target.(fmt.Stringer); ok { 362 | return s.String() 363 | } 364 | return "MISSING" 365 | } 366 | 367 | func (b *Build) commandAction(name string, checksums []string, c *commandWrapper) func() error { 368 | return func() error { 369 | s := struct { 370 | Command, Checksum, Name string 371 | ChecksumFiles string 372 | }{ 373 | Command: c.command.Shell(), 374 | Checksum: c.Checksum(), 375 | Name: name, 376 | ChecksumFiles: strings.Join(checksums, "\n"), 377 | } 378 | cm, err := render(cmdTpl, s) 379 | if err != nil { 380 | return err 381 | } 382 | wg := &sync.WaitGroup{} 383 | defer wg.Wait() 384 | ec, err := b.Target.Command(cm) 385 | if err != nil { 386 | return err 387 | } 388 | o, err := ec.StdoutPipe() 389 | if err != nil { 390 | return err 391 | } 392 | e, err := ec.StderrPipe() 393 | if err != nil { 394 | return err 395 | } 396 | wg.Add(2) 397 | l := b.maxLength 398 | if l > maxKeyLogLength { 399 | l = maxKeyLogLength 400 | } 401 | if len(name) > l { 402 | name = midTrunc(name, maxKeyLogLength) 403 | } 404 | prefix := fmt.Sprintf("%s [%-*s]", b.Target.String(), l, name) 405 | go consumeStream(prefix, gocli.Red, e, wg) 406 | go consumeStream(prefix, func(in string) string { return in }, o, wg) 407 | fmt.Println(prefix + " " + c.LogMsg()) 408 | if err := ec.Start(); err != nil { 409 | return err 410 | } 411 | return ec.Wait() 412 | } 413 | } 414 | 415 | func consumeStream(prefix string, form func(string) string, in io.Reader, wg *sync.WaitGroup) error { 416 | defer wg.Done() 417 | scanner := bufio.NewScanner(in) 418 | 419 | for scanner.Scan() { 420 | fields := strings.Split(scanner.Text(), "\t") 421 | if len(fields) > 2 { 422 | fmt.Printf("%s %s\n", prefix, form(strings.Join(fields[2:], "\t"))) 423 | } else { 424 | fmt.Printf("%s %s\n", prefix, form(scanner.Text())) 425 | } 426 | } 427 | return scanner.Err() 428 | 429 | } 430 | 431 | func render(t string, i interface{}) (string, error) { 432 | tpl, err := template.New(t).Parse(t) 433 | if err != nil { 434 | return "", err 435 | } 436 | buf := &bytes.Buffer{} 437 | err = tpl.Execute(buf, i) 438 | return buf.String(), err 439 | } 440 | 441 | const cmdTpl = `set -e 442 | 443 | function iso8601 { 444 | TZ=UTC date --iso-8601=ns | cut -d "+" -f 1 445 | } 446 | 447 | sudo_prefix="" 448 | if [[ $(id -u) != 0 ]]; then 449 | sudo_prefix="sudo" 450 | fi 451 | 452 | build_date=$(TZ=utc date +"%Y%m%d_%H%M%S") 453 | dir=/var/lib/urknall/{{ .Name }}/build.$build_date 454 | 455 | $sudo_prefix mkdir -p $dir 456 | 457 | $sudo_prefix tee $dir/{{ .Checksum }}.sh > /dev/null <<"UKEOF" 458 | {{ .Command }} 459 | UKEOF 460 | 461 | done_path=$dir/{{ .Checksum }}.done 462 | run_path=$dir/$build_date.run 463 | log_path=$dir/{{ .Checksum }}.log 464 | uk_path=/var/lib/urknall/{{ .Name }} 465 | 466 | $sudo_prefix bash $dir/{{ .Checksum }}.sh 2> >(while read line; do echo "$(iso8601) stderr $line"; done | $sudo_prefix tee -a $log_path) > >(while read line; do echo "$(iso8601) stdout $line"; done | $sudo_prefix tee -a $log_path) 467 | 468 | $sudo_prefix mv $dir/{{ .Checksum }}.sh $dir/{{ .Checksum }}.done 469 | $sudo_prefix tee $run_path > /dev/null < %s\n#!/bin/sh\nset -e\nset -x\n\n%s\n%s\nEOSCRIPT\n", targetFile, env, runner.command.Shell()) 87 | c, e := runner.build.prepareInternalCommand(rawCmd) 88 | if e != nil { 89 | return e 90 | } 91 | 92 | return c.Run() 93 | } 94 | 95 | func logError(e error) { 96 | log.Printf("ERROR: %s", e.Error()) 97 | } 98 | 99 | func (runner *commandRunner) forwardStream(logs chan <- string, stream string, wg *sync.WaitGroup, r io.Reader) { 100 | defer wg.Done() 101 | 102 | m := message("task.io", runner.build.hostname(), runner.taskName) 103 | m.Message = runner.command.Shell() 104 | if logger, ok := runner.command.(cmd.Logger); ok { 105 | m.Message = logger.Logging() 106 | } 107 | m.Stream = stream 108 | 109 | scanner := bufio.NewScanner(r) 110 | for scanner.Scan() { 111 | m.Line = scanner.Text() 112 | if m.Line == "" { 113 | m.Line = " " // empty string would be printed differently therefore add some whitespace 114 | } 115 | m.TotalRuntime = time.Since(runner.commandStarted) 116 | m.Publish(stream) 117 | logs <- time.Now().UTC().Format(time.RFC3339Nano) + "\t" + stream + "\t" + scanner.Text() 118 | } 119 | } 120 | 121 | func (runner *commandRunner) newLogWriter(path string, errors chan <- error) chan <- string { 122 | logs := make(chan string) 123 | 124 | go func() { 125 | defer close(errors) 126 | 127 | cmd, err := runner.writeLogs(path, errors, logs) 128 | switch { 129 | case err != nil: 130 | errors <- err 131 | default: 132 | if err := cmd.Wait(); err != nil { 133 | errors <- err 134 | } 135 | } 136 | }() 137 | 138 | return logs 139 | } 140 | 141 | func (runner *commandRunner) writeLogs(path string, errors chan <- error, logs <-chan string) (target.ExecCommand, error) { 142 | // so ugly, but: sudo not required and "sh -c" adds some escaping issues with the variables. This is why Command is called directly. 143 | cmd, err := runner.build.Command("cat - > " + path) 144 | if err != nil { 145 | return nil, err 146 | } 147 | 148 | // Get pipe to stdin of the execute command. 149 | in, err := cmd.StdinPipe() 150 | if err != nil { 151 | return nil, err 152 | } 153 | defer in.Close() 154 | 155 | // Run command, writing everything coming from stdin to a file. 156 | if err := cmd.Start(); err != nil { 157 | in.Close() 158 | return nil, err 159 | } 160 | 161 | // Send all messages from logs to the stdin of the new session. 162 | for log := range logs { 163 | if _, err = io.WriteString(in, log + "\n"); err != nil { 164 | errors <- err 165 | } 166 | } 167 | return cmd, err 168 | 169 | } 170 | -------------------------------------------------------------------------------- /constants.go: -------------------------------------------------------------------------------- 1 | package urknall 2 | 3 | const ( 4 | ukGROUP = "urknall" // Group used for cache file ownership on host. 5 | ukCACHEDIR = "/var/lib/urknall" // Directory for urknall cache. 6 | maxKeyLogLength = 48 // pick something fixed for now 7 | ) 8 | -------------------------------------------------------------------------------- /examples/Makefile: -------------------------------------------------------------------------------- 1 | build: 2 | go get . 3 | -------------------------------------------------------------------------------- /examples/cmd_add_user.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "fmt" 4 | 5 | // AddUser adds a new linux user (normal or system user) if it does not exist already 6 | func AddUser(name string, systemUser bool) *ShellCommand { 7 | testForUser := "id " + name + " 2>&1 > /dev/null" 8 | userAddOpts := "" 9 | if systemUser { 10 | userAddOpts = "--system" 11 | } else { 12 | userAddOpts = "-m -s /bin/bash" 13 | } 14 | return Or(testForUser, fmt.Sprintf("useradd %s %s", userAddOpts, name)) 15 | } 16 | -------------------------------------------------------------------------------- /examples/cmd_as_user.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "fmt" 4 | 5 | // Convenience function to run a command as a certain user. Setting an empty user will do nothing, as the command is 6 | // then executed as "root". Note that nested calls will not work. The function will panic if it detects such a scenario. 7 | func AsUser(user string, i interface{}) *ShellCommand { 8 | switch c := i.(type) { 9 | case *ShellCommand: 10 | if c.isExecutedAsUser() { 11 | panic(`nesting "AsUser" calls not supported`) 12 | } 13 | c.user = user 14 | return c 15 | case string: 16 | return &ShellCommand{Command: c, user: user} 17 | default: 18 | panic(fmt.Sprintf(`type "%T" not supported`, c)) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /examples/cmd_bool.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | // Combine the given commands with "and", i.e. all commands must succeed. Execution is stopped immediately if one of the 9 | // commands fails, the subsequent ones are not executed! If only one command is given nothing happens. 10 | func And(cmd interface{}, cmds ...interface{}) *ShellCommand { 11 | cs := mergeSubCommands(cmd, cmds...) 12 | 13 | finalCommand := fmt.Sprintf("{ %s; }", strings.Join(cs, " && ")) 14 | if len(cs) == 1 { 15 | finalCommand = cs[0] 16 | } 17 | return &ShellCommand{Command: finalCommand} 18 | } 19 | 20 | // Combine the given commands with "or", i.e. try one after one, untill the first returns success. If only a single 21 | // command is given, nothing happens. 22 | func Or(cmd interface{}, cmds ...interface{}) *ShellCommand { 23 | cs := mergeSubCommands(cmd, cmds...) 24 | 25 | finalCommand := fmt.Sprintf("{ %s; }", strings.Join(cs, " || ")) 26 | if len(cs) == 1 { 27 | finalCommand = cs[0] 28 | } 29 | return &ShellCommand{Command: finalCommand} 30 | } 31 | 32 | // If the tests succeeds run the given command. The test must be based on bash's test syntax (see "man test"). Just 33 | // state what should be given, like for example "-f /tmp/foo", to state that the file (-f) "/tmp/foo" must exist. 34 | // 35 | // Note that this is a double-edged sword, perfectly fit to hurt yourself. Take the following example: 36 | // [[ -f /tmp/foo ]] && echo "file exists" && exit 1 37 | // The intention is to fail if a certain file exists. The problem is that this doesn't work out. The command must return 38 | // a positive return value if the file does not exit, but it won't. Use the "IfNot" method like in this statement: 39 | // [[ ! -f /tmp/foo ]] || { echo "file exists" && exit 1; } 40 | func If(test string, i interface{}) *ShellCommand { 41 | if test == "" { 42 | panic("empty test given") 43 | } 44 | 45 | baseCommand := "{ [[ %s ]] && %s; }" 46 | 47 | switch cmd := i.(type) { 48 | case *ShellCommand: 49 | cmd.Command = fmt.Sprintf(baseCommand, test, cmd.Command) 50 | return cmd 51 | case string: 52 | if cmd == "" { 53 | panic("empty command given") 54 | } 55 | return &ShellCommand{Command: fmt.Sprintf(baseCommand, test, cmd)} 56 | default: 57 | panic(fmt.Sprintf(`type "%T" not supported`, cmd)) 58 | } 59 | } 60 | 61 | // If the tests does not succeed run the given command. The tests must be based on bash's test syntax (see "man test"). 62 | func IfNot(test string, i interface{}) *ShellCommand { 63 | if test == "" { 64 | panic("empty test given") 65 | } 66 | 67 | baseCommand := "{ [[ %s ]] || %s; }" 68 | 69 | switch cmd := i.(type) { 70 | case *ShellCommand: 71 | cmd.Command = fmt.Sprintf(baseCommand, test, cmd.Command) 72 | return cmd 73 | case string: 74 | if cmd == "" { 75 | panic("empty command given") 76 | } 77 | return &ShellCommand{Command: fmt.Sprintf(baseCommand, test, cmd)} 78 | default: 79 | panic(fmt.Sprintf(`type "%T" not supported`, cmd)) 80 | } 81 | } 82 | 83 | func mergeSubCommands(cmd interface{}, cmds ...interface{}) (cs []string) { 84 | cmdList := make([]interface{}, 0, len(cmds)+1) 85 | cmdList = append(cmdList, cmd) 86 | cmdList = append(cmdList, cmds...) 87 | 88 | for i := range cmdList { 89 | switch cmd := cmdList[i].(type) { 90 | case *ShellCommand: 91 | if cmd.user != "" && cmd.user != "root" { 92 | panic("AsUser not supported in nested commands") 93 | } 94 | cs = append(cs, cmd.Command) 95 | case string: 96 | if cmd == "" { // ignore empty commands 97 | panic("empty command found") 98 | } 99 | cs = append(cs, cmd) 100 | default: 101 | panic(fmt.Sprintf(`type "%T" not supported`, cmd)) 102 | } 103 | } 104 | return cs 105 | } 106 | -------------------------------------------------------------------------------- /examples/cmd_download.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path" 7 | "strings" 8 | 9 | "github.com/dynport/urknall/utils" 10 | ) 11 | 12 | const TMP_DOWNLOAD_DIR = "/tmp/downloads" 13 | 14 | // Download the URL and write the file to the given destination, with owner and permissions set accordingly. 15 | // Destination can either be an existing directory or a file. If a directory is given the downloaded file will moved 16 | // there using the file name from the URL. If it is a file, the downloaded file will be moved (and possibly renamed) to 17 | // that destination. If the extract flag is set the downloaded file will be extracted to the directory given in the 18 | // destination field. 19 | type DownloadCommand struct { 20 | Url string // Where to download from. 21 | Destination string // Where to put the downloaded file. 22 | Owner string // Owner of the downloaded file. 23 | Permissions os.FileMode // Permissions of the downloaded file. 24 | Extract bool // Extract the downloaded archive. 25 | } 26 | 27 | // Download the file from the given URL and extract it to the given directory. If the directory does not exist it is 28 | // created. See the "ExtractFile" command for a list of supported archive types. 29 | func DownloadAndExtract(url, destination string) *DownloadCommand { 30 | return &DownloadCommand{Url: url, Destination: destination, Extract: true} 31 | } 32 | 33 | func Download(url, destination, owner string, permissions os.FileMode) *DownloadCommand { 34 | return &DownloadCommand{Url: url, Destination: destination, Owner: owner, Permissions: permissions} 35 | } 36 | 37 | func (cmd *DownloadCommand) Validate() error { 38 | if cmd.Url == "" { 39 | return fmt.Errorf("Url must be set") 40 | } 41 | if cmd.Destination == "" { 42 | return fmt.Errorf("Destination to download %q to must be set", cmd.Url) 43 | } 44 | return nil 45 | } 46 | 47 | func (cmd *DownloadCommand) Render(i interface{}) { 48 | cmd.Url = utils.MustRenderTemplate(cmd.Url, i) 49 | cmd.Destination = utils.MustRenderTemplate(cmd.Destination, i) 50 | } 51 | 52 | func (dc *DownloadCommand) Shell() string { 53 | filename := path.Base(dc.Url) 54 | destination := fmt.Sprintf("%s/%s", TMP_DOWNLOAD_DIR, filename) 55 | 56 | cmd := []string{} 57 | 58 | cmd = append(cmd, fmt.Sprintf("which curl > /dev/null || { apt-get update && apt-get install -y curl; }")) 59 | cmd = append(cmd, fmt.Sprintf("mkdir -p %s", TMP_DOWNLOAD_DIR)) 60 | cmd = append(cmd, fmt.Sprintf("cd %s", TMP_DOWNLOAD_DIR)) 61 | cmd = append(cmd, fmt.Sprintf(`curl -SsfLO "%s"`, dc.Url)) 62 | 63 | switch { 64 | case dc.Extract && dc.Destination == "": 65 | panic(fmt.Errorf("shall extract, but don't know where (i.e. destination field is empty")) 66 | case dc.Extract: 67 | cmd = append(cmd, Extract(destination, dc.Destination).Shell()) 68 | case dc.Destination != "": 69 | cmd = append(cmd, fmt.Sprintf("mv %s %s", destination, dc.Destination)) 70 | destination = dc.Destination 71 | } 72 | 73 | if dc.Owner != "" && dc.Owner != "root" { 74 | ifFile := fmt.Sprintf("{ if [ -f %s ]; then chown %s %s; fi; }", destination, dc.Owner, destination) 75 | ifInDir := fmt.Sprintf("{ if [ -d %s && -f %s/%s ]; then chown %s %s/%s; fi; }", destination, destination, filename, dc.Owner, destination, filename) 76 | ifDir := fmt.Sprintf("{ if [ -d %s ]; then chown -R %s %s; fi; }", destination, dc.Owner, destination) 77 | err := `{ echo "Couldn't determine target" && exit 1; }` 78 | cmd = append(cmd, fmt.Sprintf("{ %s; }", strings.Join([]string{ifFile, ifInDir, ifDir, err}, " || "))) 79 | } 80 | 81 | if dc.Permissions != 0 { 82 | ifFile := fmt.Sprintf("{ if [ -f %s ]; then chmod %o %s; fi; }", destination, dc.Permissions, destination) 83 | ifInDir := fmt.Sprintf("{ if [ -d %s && -f %s/%s ]; then chmod %o %s/%s; fi; }", destination, destination, 84 | filename, dc.Permissions, destination, filename) 85 | ifDir := fmt.Sprintf("{ if [ -d %s ]; then chmod %o %s; fi; }", destination, dc.Permissions, destination) 86 | err := `{ echo "Couldn't determine target" && exit 1; }` 87 | cmd = append(cmd, fmt.Sprintf("{ %s; }", strings.Join([]string{ifFile, ifInDir, ifDir, err}, " || "))) 88 | } 89 | 90 | return strings.Join(cmd, " && ") 91 | } 92 | 93 | func (dc *DownloadCommand) Logging() string { 94 | sList := []string{"[DWNLOAD]"} 95 | 96 | if dc.Owner != "" && dc.Owner != "root" { 97 | sList = append(sList, fmt.Sprintf("[CHOWN:%s]", dc.Owner)) 98 | } 99 | 100 | if dc.Permissions != 0 { 101 | sList = append(sList, fmt.Sprintf("[CHMOD:%.4o]", dc.Permissions)) 102 | } 103 | 104 | sList = append(sList, fmt.Sprintf(" >> downloading file %q", dc.Url)) 105 | if dc.Extract { 106 | sList = append(sList, " and extracting archive") 107 | } 108 | if dc.Destination != "" { 109 | sList = append(sList, fmt.Sprintf(" to %q", dc.Destination)) 110 | } 111 | return strings.Join(sList, "") 112 | } 113 | -------------------------------------------------------------------------------- /examples/cmd_extract_file.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "path" 6 | "strings" 7 | ) 8 | 9 | // Extract the file at the given directory. The following file extensions are currently supported (".tar", ".tgz", 10 | // ".tar.gz", ".tbz", ".tar.bz2" for tar archives, and ".zip" for zipfiles). 11 | func Extract(file, targetDir string) *ShellCommand { 12 | if targetDir == "" { 13 | panic("empty target directory given") 14 | } 15 | 16 | var extractCmd *ShellCommand 17 | switch { 18 | case strings.HasSuffix(file, ".tar"): 19 | extractCmd = extractTarArchive(file, targetDir, "") 20 | case strings.HasSuffix(file, ".tgz"): 21 | fallthrough 22 | case strings.HasSuffix(file, ".tar.gz"): 23 | extractCmd = extractTarArchive(file, targetDir, "gz") 24 | case strings.HasSuffix(file, ".tbz"): 25 | fallthrough 26 | case strings.HasSuffix(file, ".tar.bz2"): 27 | extractCmd = extractTarArchive(file, targetDir, "bz2") 28 | case strings.HasSuffix(file, ".zip"): 29 | extractCmd = &ShellCommand{Command: fmt.Sprintf("unzip -d %s %s", targetDir, file)} 30 | default: 31 | panic(fmt.Sprintf("type of file %q not a supported archive", path.Base(file))) 32 | } 33 | 34 | return And( 35 | Mkdir(targetDir, "", 0), 36 | extractCmd) 37 | } 38 | 39 | func extractTarArchive(path, targetDir, compression string) *ShellCommand { 40 | additionalCommand := "" 41 | switch compression { 42 | case "gz": 43 | additionalCommand = "z" 44 | case "bz2": 45 | additionalCommand = "j" 46 | } 47 | return And( 48 | fmt.Sprintf("cd %s", targetDir), 49 | fmt.Sprintf("tar xf%s %s", additionalCommand, path)) 50 | } 51 | -------------------------------------------------------------------------------- /examples/cmd_file.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "crypto/sha1" 7 | "crypto/sha256" 8 | "encoding/base64" 9 | "encoding/hex" 10 | "fmt" 11 | "io" 12 | "os" 13 | "path/filepath" 14 | "strings" 15 | 16 | "github.com/dynport/urknall/utils" 17 | ) 18 | 19 | // The "FileCommand" is used to write files to the host being provisioned. The go templating mechanism (see 20 | // http://golang.org/pkg/text/template) is applied on the file's content using the package. Thereby it is possible to 21 | // have dynamic content (based on the package's configuration) for the file content and at the same time store it in 22 | // an asset (which is generated at compile time). Please note that the underlying actions will panic if either no path 23 | // or content are given. 24 | type FileCommand struct { 25 | Path string // Path to the file to create. 26 | Content string // Content of the file to create. 27 | Owner string // Owner of the file to create (root per default). 28 | Permissions os.FileMode // Permissions of the file created (only changed from system default if set). 29 | } 30 | 31 | func (cmd *FileCommand) Render(i interface{}) { 32 | cmd.Path = utils.MustRenderTemplate(cmd.Path, i) 33 | cmd.Content = utils.MustRenderTemplate(cmd.Content, i) 34 | } 35 | 36 | func (cmd *FileCommand) Validate() error { 37 | if cmd.Path == "" { 38 | return fmt.Errorf("no path given") 39 | } 40 | 41 | if cmd.Content == "" { 42 | return fmt.Errorf("no content given for file %q", cmd.Path) 43 | } 44 | 45 | return nil 46 | } 47 | 48 | // Helper method to create a file at the given path with the given content, and with owner and permissions set 49 | // accordingly. The "Owner" and "Permissions" options are optional in the sense that they are ignored if set to go's 50 | // default value. 51 | func WriteFile(path string, content string, owner string, permissions os.FileMode) *FileCommand { 52 | return &FileCommand{Path: path, Content: content, Owner: owner, Permissions: permissions} 53 | } 54 | 55 | var b64 = base64.StdEncoding 56 | 57 | func (fc *FileCommand) Shell() string { 58 | buf := &bytes.Buffer{} 59 | 60 | // Zip the content. 61 | zipper := gzip.NewWriter(buf) 62 | zipper.Write([]byte(fc.Content)) 63 | zipper.Flush() 64 | zipper.Close() 65 | 66 | // Encode the zipped content in Base64. 67 | encoded := b64.EncodeToString(buf.Bytes()) 68 | 69 | // Compute sha256 hash of the encoded and zipped content. 70 | hash := sha256.New() 71 | hash.Write([]byte(fc.Content)) 72 | 73 | // Create temporary filename (hash as filename). 74 | tmpPath := fmt.Sprintf("/tmp/wunderscale.%x", hash.Sum(nil)) 75 | 76 | // Get directory part of target file. 77 | dir := filepath.Dir(fc.Path) 78 | 79 | // Create command, that will decode and unzip the content and write to the temporary file. 80 | cmd := "" 81 | cmd += fmt.Sprintf("mkdir -p %s", dir) 82 | cmd += fmt.Sprintf(" && echo %s | base64 -d | gunzip > %s", encoded, tmpPath) 83 | if fc.Owner != "" { // If owner given, change accordingly. 84 | cmd += fmt.Sprintf(" && chown %s %s", fc.Owner, tmpPath) 85 | } 86 | if fc.Permissions > 0 { // If mode given, change accordingly. 87 | cmd += fmt.Sprintf(" && chmod %o %s", fc.Permissions, tmpPath) 88 | } 89 | // Move the temporary file to the requested location. 90 | cmd += fmt.Sprintf(" && mv %s %s", tmpPath, fc.Path) 91 | return cmd 92 | } 93 | 94 | func (fc *FileCommand) Logging() string { 95 | sList := []string{"[FILE ]"} 96 | 97 | if fc.Owner != "" && fc.Owner != "root" { 98 | sList = append(sList, fmt.Sprintf("[CHOWN:%s]", fc.Owner)) 99 | } 100 | 101 | if fc.Permissions != 0 { 102 | sList = append(sList, fmt.Sprintf("[CHMOD:%.4o]", fc.Permissions)) 103 | } 104 | 105 | sList = append(sList, " "+fc.Path) 106 | 107 | cLen := len(fc.Content) 108 | if cLen > 50 { 109 | cLen = 50 110 | } 111 | //sList = append(sList, fmt.Sprintf(" << %s", strings.Replace(string(fc.Content[0:cLen]), "\n", "⁋", -1))) 112 | return strings.Join(sList, "") 113 | } 114 | 115 | type FileSendCommand struct { 116 | Source string 117 | Target string 118 | Owner string 119 | Permissions os.FileMode 120 | } 121 | 122 | func SendFile(source, target, owner string, perm os.FileMode) *FileSendCommand { 123 | return &FileSendCommand{ 124 | Source: source, 125 | Target: target, 126 | Owner: owner, 127 | Permissions: perm, 128 | } 129 | } 130 | 131 | func (fsc *FileSendCommand) Render(i interface{}) { 132 | fsc.Source = utils.MustRenderTemplate(fsc.Source, i) 133 | fsc.Target = utils.MustRenderTemplate(fsc.Target, i) 134 | } 135 | 136 | func (fsc *FileSendCommand) Validate() error { 137 | if fsc.Source == "" { 138 | return fmt.Errorf("no source path given") 139 | } 140 | 141 | if _, e := os.Stat(fsc.Source); e != nil { 142 | return e 143 | } 144 | 145 | if fsc.Target == "" { 146 | return fmt.Errorf("no target path given for file %q", fsc.Source) 147 | } 148 | 149 | return nil 150 | } 151 | 152 | func (fsc *FileSendCommand) sourceHash() string { 153 | fh, e := os.Open(fsc.Source) 154 | if e != nil { 155 | panic(e) 156 | } 157 | defer fh.Close() 158 | 159 | hash := sha1.New() 160 | if _, e = io.Copy(hash, fh); e != nil { 161 | panic(e) 162 | } 163 | 164 | return hex.EncodeToString(hash.Sum(nil)) 165 | } 166 | 167 | func (fsc *FileSendCommand) Shell() string { 168 | sList := []string{ 169 | fmt.Sprintf("echo %q", fsc.sourceHash()), // nope use content hash 170 | fmt.Sprintf("cat - > %s", fsc.Target), 171 | } 172 | 173 | if fsc.Owner != "root" { 174 | sList = append(sList, fmt.Sprintf("chown %s %s", fsc.Owner, fsc.Target)) 175 | } 176 | sList = append(sList, fmt.Sprintf("chmod %s %s", fsc.Permissions, fsc.Target)) 177 | return strings.Join(sList, " && ") 178 | } 179 | 180 | func (fsc *FileSendCommand) Input() io.ReadCloser { 181 | fh, e := os.Open(fsc.Source) 182 | if e != nil { 183 | panic(e) 184 | } 185 | return fh 186 | } 187 | 188 | func (fsc *FileSendCommand) Logging() string { 189 | sList := []string{"[FILE ]"} 190 | 191 | if fsc.Owner != "" && fsc.Owner != "root" { 192 | sList = append(sList, fmt.Sprintf("[CHOWN:%s]", fsc.Owner)) 193 | } 194 | 195 | if fsc.Permissions != 0 { 196 | sList = append(sList, fmt.Sprintf("[CHMOD:%.4o]", fsc.Permissions)) 197 | } 198 | 199 | sList = append(sList, fmt.Sprintf(" Writing local file %s to %s", fsc.Source, fsc.Target)) 200 | 201 | return strings.Join(sList, "") 202 | } 203 | -------------------------------------------------------------------------------- /examples/cmd_fileutils.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | ) 7 | 8 | // Create the given directory with the owner and file permissions set accordingly. If the last two options are set to 9 | // go's default values nothing is done. 10 | func Mkdir(path, owner string, permissions os.FileMode) *ShellCommand { 11 | if path == "" { 12 | panic("empty path given to mkdir") 13 | } 14 | 15 | mkdirCmd := fmt.Sprintf("mkdir -p %s", path) 16 | 17 | optsCmds := make([]interface{}, 0, 2) 18 | if owner != "" { 19 | optsCmds = append(optsCmds, fmt.Sprintf("chown %s %s", owner, path)) 20 | } 21 | 22 | if permissions != 0 { 23 | optsCmds = append(optsCmds, fmt.Sprintf("chmod %o %s", permissions, path)) 24 | } 25 | 26 | return And(mkdirCmd, optsCmds...) 27 | } 28 | -------------------------------------------------------------------------------- /examples/cmd_shell.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/dynport/urknall/utils" 8 | ) 9 | 10 | // A shell command is just that: something that is executed in a shell on the host to be provisioned. There is quite a 11 | // lot of infrastructure to build such commands. To make construction of complicated commands easier those helpers use 12 | // the most generic type "interface{}". Thereby it is possible to use these functions with "strings" or other 13 | // "ShellCommands" (returned by other helpers for example). 14 | // 15 | // There are some commands that relate to the system's package management. Those are currently based on apt, i.e. only 16 | // debian based systems can be used (our current system of choice is ubuntu server in version 12.04LTS as of this 17 | // writing). 18 | type ShellCommand struct { 19 | Command string // Command to be executed in the shell. 20 | user string // User to run the command as. 21 | } 22 | 23 | func (cmd *ShellCommand) Render(i interface{}) { 24 | cmd.Command = utils.MustRenderTemplate(cmd.Command, i) 25 | if cmd.user != "" { 26 | cmd.user = utils.MustRenderTemplate(cmd.user, i) 27 | } 28 | } 29 | 30 | func Shell(cmd string) *ShellCommand { 31 | return &ShellCommand{Command: cmd} 32 | } 33 | 34 | func (sc *ShellCommand) Shell() string { 35 | if sc.isExecutedAsUser() { 36 | return fmt.Sprintf("su -l %s <&2 && exit 1; }; }", 13 | int64(t), path, int64(t), path) 14 | return &ShellCommand{ 15 | Command: cmd, 16 | } 17 | } 18 | 19 | // Wait for the given unix file socket to appear. Break and fail if it doesn't appear after the given number of seconds. 20 | func WaitForUnixSocket(path string, timeout time.Duration) *ShellCommand { 21 | t := 10 * timeout.Seconds() 22 | cmd := fmt.Sprintf( 23 | "x=0; while ((x<%d)) && ! { netstat -lx | grep \"%s$\"; }; do x=\\$((x+1)); sleep .1; done && { ((x<%d)) || { echo \"socket %s did not appear\" 1>&2 && exit 1; }; }", 24 | int64(t), path, int64(t), path) 25 | return &ShellCommand{ 26 | Command: cmd, 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /examples/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "os" 6 | 7 | "github.com/dynport/urknall" 8 | ) 9 | 10 | var logger = log.New(os.Stderr, "", 0) 11 | 12 | func main() { 13 | if e := run(); e != nil { 14 | logger.Fatal(e) 15 | } 16 | } 17 | 18 | type Template struct { 19 | } 20 | 21 | func (tpl *Template) Render(p urknall.Package) { 22 | p.AddCommands("hello", Shell("echo hello world")) 23 | } 24 | 25 | func run() error { 26 | defer urknall.OpenLogger(os.Stdout).Close() 27 | var target urknall.Target 28 | var e error 29 | uri := "ubuntu@my.host" 30 | password := "" 31 | if password != "" { 32 | target, e = urknall.NewSshTargetWithPassword(uri, password) 33 | } else { 34 | target, e = urknall.NewSshTarget(uri) 35 | } 36 | if e != nil { 37 | return e 38 | } 39 | return urknall.Run(target, &Template{}) 40 | } 41 | -------------------------------------------------------------------------------- /examples/tpl_cronjob.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "github.com/dynport/urknall" 6 | ) 7 | 8 | type Cronjob struct { 9 | // Name of the cron job. Used for file names and logging. 10 | Name string `urknall:"required=true"` 11 | // The script to be executed. 12 | Script []byte `urknall:"required=true"` 13 | // The common cron job pattern, i.e. something like '*/5 * * * *' for every five minutes. 14 | Pattern string `urknall:"required=true"` 15 | // File mode of the script added. 16 | Mode int 17 | } 18 | 19 | func (job *Cronjob) Render(r urknall.Package) { 20 | mode := os.FileMode(job.Mode) 21 | if mode == 0 { 22 | mode = 0755 23 | } 24 | scriptPath := "/opt/cron/" + job.Name 25 | cronPath := "/etc/cron.d/" + job.Name 26 | 27 | r.AddCommands("script", WriteFile(scriptPath, string(job.Script), "root", mode)) 28 | r.AddCommands("cron", WriteFile(cronPath, job.Pattern + " root " + scriptPath + " 2>&1 | logger -i -t " + job.Name + "\n", "root", 0644)) 29 | } -------------------------------------------------------------------------------- /examples/tpl_docker.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "github.com/dynport/urknall" 4 | 5 | type Docker struct { 6 | Version string `urknall:"required=true"` // e.g. 1.1.0 7 | CustomInstallDir string 8 | Public bool 9 | } 10 | 11 | func (docker *Docker) Render(pkg urknall.Package) { 12 | pkg.AddCommands("packages", InstallPackages("aufs-tools", "cgroup-lite", "xz-utils", "git", "linux-image-extra-$(uname -r)")) 13 | pkg.AddCommands("install", 14 | Mkdir("{{ .InstallDir }}/bin", "root", 0755), 15 | Download("http://get.docker.io/builds/Linux/x86_64/docker-{{ .Version }}", "{{ .InstallDir }}/bin/docker", "root", 0755), 16 | ) 17 | pkg.AddCommands("upstart", WriteFile("/etc/init/docker.conf", dockerUpstart, "root", 0644)) 18 | } 19 | 20 | const dockerUpstart = `exec {{ .InstallDir }}/bin/docker -d -H tcp://{{ if .Public }}0.0.0.0{{ else }}127.0.0.1{{ end }}:4243 -H unix:///var/run/docker.sock 2>&1 | logger -i -t docker 21 | ` 22 | 23 | func (docker *Docker) InstallDir() string { 24 | if docker.Version == "" { 25 | panic("Version must be set") 26 | } 27 | if docker.CustomInstallDir != "" { 28 | return docker.CustomInstallDir 29 | } 30 | return "/opt/docker-" + docker.Version 31 | } 32 | -------------------------------------------------------------------------------- /examples/tpl_elasticsearch.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "github.com/dynport/urknall" 4 | 5 | type ElasticSearch struct { 6 | Version string `urknall:"required=true"` 7 | ClusterName string `urknall:"default=elasticsearch"` 8 | DataPath string `urknall:"default=/data/elasticsearch"` 9 | 10 | // optional 11 | SyslogHost string 12 | DiscoveryHosts string 13 | LogPath string 14 | NodeName string 15 | } 16 | 17 | func (es *ElasticSearch) Render(pkg urknall.Package) { 18 | pkg.AddCommands("java7", 19 | InstallPackages("openjdk-7-jdk"), 20 | Shell("update-alternatives --set java /usr/lib/jvm/java-7-openjdk-amd64/jre/bin/java"), 21 | ) 22 | pkg.AddCommands("download", DownloadAndExtract("{{ .Url }}", "/opt/")) 23 | pkg.AddCommands("user", AddUser("elasticsearch", true)) 24 | pkg.AddCommands("mkdir", Mkdir(es.DataPath, "elasticsearch", 0755)) 25 | pkg.AddCommands("config", 26 | WriteFile("{{ .InstallDir }}/config/elasticsearch.yml", elasticSearchConfig, "root", 0644), 27 | WriteFile("{{ .InstallDir }}/config/logging.yml", elasticSearchConfigLogger, "root", 0644), 28 | WriteFile("/etc/init/elasticsearch.conf", elasticSearchUpstart, "root", 0644), 29 | ) 30 | } 31 | 32 | func (es *ElasticSearch) Url() string { 33 | return "https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-{{ .Version }}.tar.gz" 34 | } 35 | 36 | func (es *ElasticSearch) InstallDir() string { 37 | if es.Version == "" { 38 | panic("Version must be set") 39 | } 40 | return "/opt/elasticsearch-" + es.Version 41 | } 42 | 43 | const elasticSearchUpstart = ` 44 | {{ with .DataPath }} 45 | pre-start script 46 | mkdir -p {{ . }} 47 | end script 48 | {{ end }} 49 | 50 | exec {{ .InstallDir }}/bin/elasticsearch -f 51 | ` 52 | 53 | const elasticSearchConfigLogger = ` 54 | rootLogger: DEBUG, syslog 55 | logger: 56 | # log action execution errors for easier debugging 57 | action: DEBUG 58 | # reduce the logging for aws, too much is logged under the default INFO 59 | com.amazonaws: WARN 60 | 61 | index.search.slowlog: TRACE{{ with .SyslogHost }}, syslog{{ end }} 62 | index.indexing.slowlog: TRACE{{ with .SyslogHost }}, syslog{{ end }} 63 | 64 | additivity: 65 | index.search.slowlog: false 66 | index.indexing.slowlog: false 67 | 68 | 69 | {{ with .SyslogHost }} 70 | appender: 71 | syslog: 72 | type: syslog 73 | syslogHost: {{ . }}:514 74 | facility: local0 75 | layout: 76 | type: pattern 77 | conversionPattern: "[%-5p] [%-25c] %m%n" 78 | {{ end }} 79 | ` 80 | 81 | const elasticSearchConfig = ` 82 | path.data: {{ .DataPath }} 83 | path.logs: {{ .DataPath }}/logs 84 | {{ with .NodeName }}node.name: {{ . }}{{ end }} 85 | {{ with .ClusterName }}cluster.name: {{ . }}{{ end }} 86 | {{ with .DiscoveryHosts }}discovery.zen.ping.unicast.hosts: {{ . }}{{ end }} 87 | ` 88 | -------------------------------------------------------------------------------- /examples/tpl_firewall.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "strconv" 7 | 8 | "github.com/dynport/urknall" 9 | "github.com/dynport/urknall/cmd" 10 | ) 11 | 12 | type Firewall struct { 13 | Interface string `urknall:"default=eth0"` 14 | WithVPN bool 15 | Paranoid bool 16 | Rules []*FirewallRule 17 | IPSets []*FirewallIPSet // List of ipsets for the firewall. 18 | } 19 | 20 | func (fw *Firewall) Render(pkg urknall.Package) { 21 | var ipsetsCmd cmd.Command 22 | if len(fw.IPSets) > 0 { 23 | ipsetsCmd = WriteFile("/etc/iptables/ipsets", fwIpset, "root", 0644) 24 | } else { 25 | ipsetsCmd = Shell("rm -f /etc/iptables/ipsets") 26 | } 27 | pkg.AddCommands("base", 28 | InstallPackages("iptables", "ipset"), 29 | WriteFile("/etc/network/if-pre-up.d/iptables", firewallUpstart, "root", 0744), 30 | WriteFile("/etc/iptables/rules_ipv4", fw_rules_ipv4, "root", 0644), 31 | WriteFile("/etc/iptables/rules_ipv6", fw_rules_ipv6, "root", 0644), 32 | ipsetsCmd, 33 | Shell("{ modprobe iptable_filter && modprobe iptable_nat; }; /bin/true"), // here to make sure next command succeeds. 34 | Shell("IFACE={{ .Interface }} /etc/network/if-pre-up.d/iptables"), 35 | ) 36 | } 37 | 38 | // IPSets are the possibility to change a rule, without actually rewriting the rules. That is they add some sort of 39 | // flexibility with regard to dynamic entities like a load balancer, which must have access to the different machines 40 | // that should take the load. 41 | // 42 | // A set is defined by a name, that is used in iptables rule (see "Rule.(Source|Destination).IPSet") to reference the 43 | // contained entities. The type defines what parameters must match an entry (see "ipset --help" output and the man page 44 | // for a list of allowed values), for example a set could define hosts and ports. 45 | // 46 | // The family defines the type of IP address to handle, either IPv4 or IPv6. The allowed values are "inet" and "inet6" 47 | // respectively. 48 | // 49 | // There are some ipset internal parameters that shouldn't need to be changed often. Those are "HashSize" that defines 50 | // the size of the underlying hash. This value defaults to 1024. The "MaxElem" number determines how much elements there 51 | // can be in the set at most. 52 | // 53 | // An initial set of members can be defined, if reasonable. 54 | type FirewallIPSet struct { 55 | Name string // Name of the ipset. 56 | Type string // Type of the ipset. 57 | Family string // Network address family. 58 | HashSize int // Size of the hash. 59 | MaxElem int // Max number of elements of the set. 60 | Members []net.IP // Initial set of members. 61 | } 62 | 63 | func (ips *FirewallIPSet) IPSetRestore() (cmd string) { 64 | cmd = fmt.Sprintf("create %s %s family %s hashsize %d maxelem %d\n", 65 | ips.Name, ips.Type, ips.family(), ips.hashsize(), ips.maxelem()) 66 | for idx := range ips.Members { 67 | cmd += fmt.Sprintf("add %s %s\n", ips.Name, ips.Members[idx].String()) 68 | } 69 | return cmd + "\n" 70 | } 71 | 72 | func (ips *FirewallIPSet) family() string { 73 | if ips.Family == "" { 74 | return "inet" 75 | } 76 | return ips.Family 77 | } 78 | 79 | func (i *FirewallIPSet) hashsize() int { 80 | if i.HashSize == 0 { 81 | return 1024 82 | } 83 | return i.HashSize 84 | } 85 | 86 | func (i *FirewallIPSet) maxelem() int { 87 | if i.MaxElem == 0 { 88 | return 65536 89 | } 90 | return i.MaxElem 91 | } 92 | 93 | // A rule defines what is allowed to flow from some source to some destination. A description can be added to make the 94 | // resulting scripts more readable. 95 | // 96 | // The "Chain" field determines which chain the rule is added to. This should be either "INPUT", "OUTPUT", or "FORWARD", 97 | // with the names of the chains mostly speaking for themselves. 98 | // 99 | // The protocol setting is another easy match for the rule and especially required for some of the target's settings, 100 | // i.e. if a port is specified the protocol must be given too. 101 | // 102 | // Source and destination are the two communicating entities. For the input chain the local host is destination and for 103 | // output it is the source. 104 | type FirewallRule struct { 105 | Description string 106 | Chain string // Chain to add the rule to. 107 | Protocol string // The protocol used. 108 | 109 | Source *FirewallTarget // The source of the packet. 110 | Destination *FirewallTarget // The destination of the packet. 111 | } 112 | 113 | // Method to create something digestable for IPtablesRestore (aka users might well ignore this). 114 | func (r *FirewallRule) Filter() (cmd string) { 115 | cfg := &iptConfig{rule: r, moduleConfig: map[string]iptModConfig{}} 116 | 117 | if r.Source != nil { 118 | r.Source.convert(cfg, "src") 119 | } 120 | 121 | if r.Destination != nil { 122 | r.Destination.convert(cfg, "dest") 123 | } 124 | 125 | return cfg.FilterTableRule() 126 | } 127 | 128 | func (r *FirewallRule) isNATRule() bool { 129 | return r.Chain == "FORWARD" && 130 | ((r.Source != nil && r.Source.NAT != "") || 131 | (r.Destination != nil && r.Destination.NAT != "")) 132 | } 133 | 134 | func (r *FirewallRule) NAT() (cmd string) { 135 | if !r.isNATRule() { 136 | return "" 137 | } 138 | 139 | cfg := &iptConfig{rule: r, moduleConfig: map[string]iptModConfig{}} 140 | 141 | if r.Source != nil { 142 | r.Source.convert(cfg, "src") 143 | } 144 | 145 | if r.Destination != nil { 146 | r.Destination.convert(cfg, "dest") 147 | } 148 | 149 | return cfg.NATTableRule() 150 | } 151 | 152 | func (r *FirewallRule) IPsets() { 153 | } 154 | 155 | type iptModConfig map[string]string 156 | 157 | type iptConfig struct { 158 | rule *FirewallRule 159 | 160 | sourceIP string 161 | destIP string 162 | 163 | sourceIface string 164 | destIface string 165 | 166 | sourceNAT string 167 | destNAT string 168 | 169 | moduleConfig map[string]iptModConfig 170 | } 171 | 172 | func (cfg *iptConfig) basicSettings(natRule bool) (s string) { 173 | if cfg.rule.Protocol != "" { 174 | s += " --protocol " + cfg.rule.Protocol 175 | } 176 | if cfg.sourceIP != "" { 177 | s += " --source " + cfg.sourceIP 178 | } 179 | if cfg.sourceIface != "" { 180 | if cfg.rule.Chain == "FORWARD" { 181 | if !natRule || cfg.destNAT != "" { 182 | s += " --in-interface " + cfg.sourceIface 183 | } 184 | } else { 185 | s += " --out-interface " + cfg.sourceIface 186 | } 187 | } 188 | if cfg.destIP != "" { 189 | s += " --destination " + cfg.destIP 190 | } 191 | if cfg.destIface != "" { 192 | if cfg.rule.Chain == "FORWARD" { 193 | if !natRule || cfg.sourceNAT != "" { 194 | s += " --out-interface " + cfg.destIface 195 | } 196 | } else { 197 | s += " --in-interface " + cfg.destIface 198 | } 199 | } 200 | return s 201 | } 202 | 203 | func (cfg *iptConfig) FilterTableRule() (s string) { 204 | if cfg.rule.Description != "" { 205 | s = "# " + cfg.rule.Description + "\n" 206 | } 207 | s += "-A " + cfg.rule.Chain 208 | 209 | s += cfg.basicSettings(false) 210 | 211 | for module, modOptions := range cfg.moduleConfig { 212 | s += " " + module 213 | for option, value := range modOptions { 214 | s += " " + option + " " + value 215 | } 216 | } 217 | 218 | s += " -m state --state NEW -j ACCEPT\n" 219 | return s 220 | } 221 | 222 | func (cfg *iptConfig) NATTableRule() (s string) { 223 | if cfg.rule.Description != "" { 224 | s = "# " + cfg.rule.Description + "\n" 225 | } 226 | 227 | switch { 228 | case cfg.sourceNAT != "" && cfg.destNAT == "": 229 | s += "-A POSTROUTING" 230 | case cfg.sourceNAT == "" && cfg.destNAT != "": 231 | s += "-A PREROUTING" 232 | default: 233 | panic("but you said NAT would be configured?!") 234 | } 235 | 236 | s += cfg.basicSettings(true) 237 | 238 | switch { 239 | case cfg.sourceNAT == "MASQ": 240 | s += " -j MASQUERADE" 241 | case cfg.sourceNAT != "": 242 | s += " -j SNAT --to " + cfg.sourceNAT 243 | case cfg.destNAT != "": 244 | s += " -j DNAT --to " + cfg.destNAT 245 | } 246 | 247 | return s 248 | } 249 | 250 | func (t *FirewallTarget) convert(cfg *iptConfig, tType string) { 251 | if t.Port != 0 { 252 | if cfg.rule.Protocol == "" { 253 | panic("port requires the protocol to be specified") 254 | } 255 | 256 | module := "-m " + cfg.rule.Protocol 257 | if _, found := cfg.moduleConfig[module]; !found { 258 | cfg.moduleConfig[module] = iptModConfig{} 259 | } 260 | switch tType { 261 | case "src": 262 | cfg.moduleConfig[module]["--source-port"] = strconv.Itoa(t.Port) 263 | case "dest": 264 | cfg.moduleConfig[module]["--destination-port"] = strconv.Itoa(t.Port) 265 | } 266 | } 267 | 268 | if t.IP != nil { 269 | switch tType { 270 | case "src": 271 | cfg.sourceIP = t.IP.String() 272 | case "dest": 273 | cfg.destIP = t.IP.String() 274 | } 275 | } 276 | 277 | if t.IPSet != "" { 278 | module := "-m set" 279 | if _, found := cfg.moduleConfig[module]; !found { 280 | cfg.moduleConfig[module] = iptModConfig{} 281 | } 282 | value := cfg.moduleConfig[module]["--match-set "+t.IPSet] 283 | set := "" 284 | switch tType { 285 | case "src": 286 | set = "src" 287 | case "dest": 288 | set = "dst" 289 | } 290 | if value != "" { 291 | cfg.moduleConfig[module]["--match-set "+t.IPSet] = value + "," + set 292 | } else { 293 | cfg.moduleConfig[module]["--match-set "+t.IPSet] = set 294 | } 295 | } 296 | 297 | if t.Interface != "" { 298 | switch tType { 299 | case "src": 300 | cfg.sourceIface = t.Interface 301 | case "dest": 302 | cfg.destIface = t.Interface 303 | } 304 | } 305 | 306 | if t.NAT != "" { 307 | switch tType { 308 | case "src": // for input on the source the destination address can be modified. 309 | cfg.destNAT = t.NAT 310 | case "dest": // for output on the destination the source address can be modified. 311 | cfg.sourceNAT = t.NAT 312 | } 313 | 314 | if cfg.sourceNAT != "" && cfg.destNAT != "" { 315 | panic("only source or destination NAT allowed!") 316 | } 317 | } 318 | } 319 | 320 | // The target of a rule. It can be specified either by IP or the name of an IPSet. Additional parameters are the port 321 | // and interface used. It's totally valid to only specify a subset (or even none) of the fields. For example IP and 322 | // IPSet must not be given for the host the rule is applied on. 323 | type FirewallTarget struct { 324 | IP net.IP // IP of the target. 325 | IPSet string // IPSet used for matching. 326 | Port int // Port packets must use to match. 327 | Interface string // Interface the packet goes through. 328 | NAT string // NAT configuration (empty, "MASQ", or Interface's IP). 329 | } 330 | 331 | const fw_rules_ipv6 = ` 332 | *filter 333 | :INPUT DROP [0:0] 334 | :FORWARD DROP [0:0] 335 | :OUTPUT DROP [0:0] 336 | 337 | COMMIT 338 | ` 339 | 340 | const fw_rules_ipv4 = `*filter 341 | :INPUT DROP [0:0] 342 | :FORWARD DROP [0:0] 343 | :OUTPUT DROP [0:0] 344 | 345 | # Accept any related or established connections. 346 | -I INPUT 1 -m state --state RELATED,ESTABLISHED -j ACCEPT 347 | -I FORWARD 1 -m state --state RELATED,ESTABLISHED -j ACCEPT 348 | -I OUTPUT 1 -m state --state {{ if not .Paranoid }}NEW,{{ end }}RELATED,ESTABLISHED -j ACCEPT 349 | 350 | # Allow all traffic on the loopback interface. 351 | -A INPUT -i lo -j ACCEPT 352 | -A OUTPUT -o lo -j ACCEPT 353 | 354 | {{ if .WithVPN }} 355 | # Allow all traffic on the VPN interface. 356 | -A INPUT -i tun0 -j ACCEPT 357 | -A OUTPUT -o tun0 -j ACCEPT 358 | {{ end }} 359 | 360 | {{ if .Paranoid }} 361 | # Outbound DNS lookups 362 | -A OUTPUT -o {{ .Interface }} -p udp -m udp --dport 53 -j ACCEPT 363 | 364 | # Outbound PING requests 365 | -A OUTPUT -p icmp -j ACCEPT 366 | 367 | # Outbound Network Time Protocol (NTP) request 368 | -A OUTPUT -p udp --dport 123 --sport 123 -j ACCEPT 369 | 370 | # Allow outbound DHCP request - Some hosts (Linode) automatically assign the primary IP 371 | -A OUTPUT -p udp --dport 67:68 --sport 67:68 -m state --state NEW -j ACCEPT 372 | 373 | # Outbound HTTP 374 | -A OUTPUT -o {{ .Interface }} -p tcp -m tcp --dport 80 -m state --state NEW -j ACCEPT 375 | -A OUTPUT -o {{ .Interface }} -p tcp -m tcp --dport 443 -m state --state NEW -j ACCEPT 376 | {{ end }} 377 | 378 | # SSH 379 | -A INPUT -i {{ .Interface }} -p tcp -m tcp --dport 22 -m state --state NEW -j ACCEPT 380 | 381 | {{ range .Rules }}{{ .Filter }}{{ end }} 382 | 383 | {{ if .WithVPN }} 384 | # Outbound OpenVPN traffic (required to connect to the VPN). 385 | -A OUTPUT -o {{ .Interface }} -p tcp -m tcp --dport 1194 -m state --state NEW -j ACCEPT 386 | -A OUTPUT -o {{ .Interface }} -p udp -m udp --dport 1194 -m state --state NEW -j ACCEPT 387 | {{ end }} 388 | COMMIT 389 | 390 | *nat 391 | {{ range .Rules }}{{ .NAT }}{{ end }} 392 | COMMIT 393 | ` 394 | 395 | const fwIpset = `# IPSet configuration 396 | {{ range .IPSets }}{{ .IPSetRestore }}{{ end }}` 397 | const firewallUpstart = `#!/bin/sh 398 | set -e 399 | 400 | case "$IFACE" in 401 | {{ .Interface }}) 402 | test -e /etc/iptables/ipsets && /usr/sbin/ipset restore -! < /etc/iptables/ipsets 403 | /sbin/iptables-restore < /etc/iptables/rules_ipv4 404 | /sbin/ip6tables-restore < /etc/iptables/rules_ipv6 405 | ;; 406 | esac 407 | 408 | ` 409 | -------------------------------------------------------------------------------- /examples/tpl_golang.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/dynport/urknall" 5 | ) 6 | 7 | type Golang struct { 8 | Version string `urknall:"required=true"` 9 | } 10 | 11 | func (golang *Golang) Render(pkg urknall.Package) { 12 | pkg.AddCommands("packages", InstallPackages("build-essential", "curl", "bzr", "mercurial", "git-core")) 13 | pkg.AddCommands("mkdir", Mkdir("{{ .InstallDir }}", "root", 0755)) 14 | pkg.AddCommands("download", 15 | DownloadAndExtract("https://storage.googleapis.com/golang/go{{ .Version }}.linux-amd64.tar.gz", "{{ .InstallDir }}"), 16 | ) 17 | } 18 | 19 | func (golang *Golang) InstallDir() string { 20 | if golang.Version == "" { 21 | panic("Version must bese") 22 | } 23 | return "/opt/go-" + golang.Version 24 | } 25 | -------------------------------------------------------------------------------- /examples/tpl_haproxy.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/dynport/urknall" 8 | ) 9 | 10 | type HAProxy struct { 11 | Version string `urknall:"required=true"` 12 | } 13 | 14 | func (haproxy *HAProxy) url() string { 15 | return "http://haproxy.1wt.eu/download/{{ .MinorVersion }}/src/haproxy-{{ .Version }}.tar.gz" 16 | } 17 | 18 | func (haproxy *HAProxy) MinorVersion() string { 19 | parts := strings.Split(haproxy.Version, ".") 20 | if len(parts) == 3 { 21 | return strings.Join(parts[0:2], ".") 22 | } 23 | panic(fmt.Sprintf("unable to extract minor version from %q", haproxy.Version)) 24 | } 25 | 26 | func (haproxy *HAProxy) InstallDir() string { 27 | if haproxy.Version == "" { 28 | panic("Version must be set") 29 | } 30 | return "/opt/haproxy-" + haproxy.Version 31 | } 32 | 33 | func (haproxy *HAProxy) Render(pkg urknall.Package) { 34 | pkg.AddCommands("base", 35 | InstallPackages("curl", "build-essential", "libpcre3-dev"), 36 | Mkdir("/opt/src/", "root", 0755), 37 | DownloadAndExtract(haproxy.url(), "/opt/src/"), 38 | Mkdir("{{ .InstallDir }}/sbin", "root", 0755), 39 | Shell("cd /opt/src/haproxy-{{ .Version }} && make TARGET=linux25 USER_STATIC_PCRE=1 && cp ./haproxy {{ .InstallDir }}/sbin/"), 40 | WriteFile("/etc/init/haproxy.conf", haProxyInitScript, "root", 0755), 41 | ) 42 | } 43 | 44 | const haProxyInitScript = `description "Properly handle haproxy" 45 | 46 | start on (filesystem and net-device-up IFACE=lo) 47 | 48 | env PID_PATH=/var/run/haproxy.pid 49 | env BIN_PATH={{ .InstallDir }}/sbin/haproxy 50 | 51 | script 52 | exec /bin/bash <&1 | logger -i -t jenkins 42 | ` 43 | -------------------------------------------------------------------------------- /examples/tpl_nginx.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/dynport/urknall" 5 | "github.com/dynport/urknall/utils" 6 | ) 7 | 8 | type Nginx struct { 9 | Version string `urknall:"required=true"` // e.g. 1.4.7 10 | HeadersMoreVersion string `urknall:"default=0.24"` 11 | SyslogPatchVersion string `urknall:"default=1.3.14"` 12 | Local bool // install to /usr/local/nginx 13 | Autostart bool 14 | } 15 | 16 | func (ngx *Nginx) Render(pkg urknall.Package) { 17 | syslogPatchPath := "/tmp/nginx_syslog_patch" 18 | fileName := "syslog_{{ .SyslogPatchVersion }}.patch" 19 | pkg.AddCommands("packages", 20 | InstallPackages("build-essential", "curl", "libpcre3", "libpcre3-dev", "libssl-dev", "libpcrecpp0", "zlib1g-dev", "libgd2-xpm-dev"), 21 | ) 22 | pkg.AddCommands("download", 23 | DownloadAndExtract("{{ .Url }}", "/opt/src/"), 24 | ) 25 | pkg.AddCommands("syslog_patch", 26 | Mkdir(syslogPatchPath, "root", 0755), 27 | Download("https://raw.github.com/yaoweibin/nginx_syslog_patch/master/config", syslogPatchPath+"/config", "root", 0644), 28 | Download("https://raw.github.com/yaoweibin/nginx_syslog_patch/master/"+fileName, syslogPatchPath+"/"+fileName, "root", 0644), 29 | And( 30 | "cd /opt/src/nginx-{{ .Version }}", 31 | "patch -p1 < "+syslogPatchPath+"/"+fileName, 32 | ), 33 | ) 34 | pkg.AddCommands("more_clear_headers", 35 | DownloadAndExtract("https://github.com/agentzh/headers-more-nginx-module/archive/v{{ .HeadersMoreVersion }}.tar.gz", "/opt/src/"), 36 | ) 37 | pkg.AddCommands("build", 38 | And( 39 | "cd /opt/src/nginx-{{ .Version }}", 40 | "./configure --with-http_ssl_module --with-http_gzip_static_module --with-http_stub_status_module --with-http_spdy_module --add-module=/tmp/nginx_syslog_patch --add-module=/opt/src/headers-more-nginx-module-{{ .HeadersMoreVersion }} --prefix={{ .InstallDir }}", 41 | "make", 42 | "make install", 43 | ), 44 | ) 45 | pkg.AddCommands("upstart", 46 | WriteFile("/etc/init/nginx.conf", utils.MustRenderTemplate(nginxUpstartScript, ngx), "root", 0644), 47 | ) 48 | } 49 | 50 | func (ngx *Nginx) ConfDir() string { 51 | return ngx.InstallDir() + "/conf" 52 | } 53 | 54 | func (ngx *Nginx) InstallDir() string { 55 | if ngx.Local { 56 | return "/usr/local/nginx" 57 | } 58 | if ngx.Version == "" { 59 | panic("Version must be set") 60 | } 61 | return "/opt/nginx-" + ngx.Version 62 | } 63 | 64 | func (ngx *Nginx) BinPath() string { 65 | return ngx.InstallDir() + "/sbin/nginx" 66 | } 67 | 68 | func (ngx *Nginx) ReloadCommand() string { 69 | return utils.MustRenderTemplate("{{ . }} -t && {{ . }} -s reload", ngx.BinPath()) 70 | } 71 | 72 | const nginxUpstartScript = `# nginx 73 | 74 | description "nginx http daemon" 75 | author "George Shammas " 76 | 77 | {{ if .Autostart }} 78 | start on (filesystem and net-device-up IFACE=lo) 79 | stop on runlevel [!2345] 80 | {{ end }} 81 | 82 | env DAEMON={{ .InstallDir }}/sbin/nginx 83 | env PID=/var/run/nginx.pid 84 | 85 | respawn 86 | respawn limit 10 5 87 | #oom never 88 | 89 | pre-start script 90 | $DAEMON -t 91 | if [ $? -ne 0 ] 92 | then exit $? 93 | fi 94 | end script 95 | 96 | exec $DAEMON -g "daemon off;" 97 | ` 98 | 99 | func (ngx *Nginx) Url() string { 100 | return "http://nginx.org/download/nginx-{{ .Version }}.tar.gz" 101 | } 102 | -------------------------------------------------------------------------------- /examples/tpl_openvpn.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "github.com/dynport/urknall" 4 | 5 | type OpenVPN struct { 6 | Country string `urknall:"required=true"` 7 | Province string `urknall:"required=true"` 8 | City string `urknall:"required=true"` 9 | Org string `urknall:"required=true"` 10 | Email string `urknall:"required=true"` 11 | Name string `urknall:"required=true"` 12 | Netmask string `urknall:"required=true"` 13 | Address string `urknall:"default=10.19.0.0"` 14 | 15 | Routes []string 16 | } 17 | 18 | const openVpnPackagePath = "/opt/package_openvpn_key" 19 | 20 | func (p *OpenVPN) Render(r urknall.Package) { 21 | if len(p.Country) != 2 { 22 | panic("Country must be exactly 2 characters long") 23 | } 24 | if len(p.Province) != 2 { 25 | panic("Province must be exactly 2 characters long") 26 | } 27 | r.AddCommands("packages", InstallPackages("openvpn", "iptables", "zip", "easy-rsa")) 28 | r.AddCommands("easy-rsa", Shell("cp -R /usr/share/easy-rsa/ /etc/openvpn/easy-rsa/")) 29 | r.AddCommands("config", 30 | WriteFile("/etc/openvpn/easy-rsa/vars", openVpnVars, "root", 0644), 31 | Shell("ln -nfs /etc/openvpn/easy-rsa/openssl-1.0.0.cnf /etc/openvpn/easy-rsa/openssl.cnf"), 32 | ) 33 | 34 | r.AddCommands("ca", 35 | Shell(`bash -c "cd /etc/openvpn/easy-rsa && source ./vars && ./clean-all && ./pkitool --initca && ./pkitool --server {{ .Name }} && ./build-dh"`), 36 | Shell(`bash -c "cd /etc/openvpn/easy-rsa/keys && cp -v {{ .Name }}.{crt,key} ca.crt dh1024.pem /etc/openvpn/"`), 37 | ) 38 | r.AddCommands("server-config", 39 | WriteFile("/etc/openvpn/server.conf", openvpnServerConfig, "root", 0644), 40 | Shell("{ /etc/init.d/openvpn status && /etc/init.d/openvpn restart; } || /etc/init.d/openvpn start"), 41 | ) 42 | r.AddCommands("scripts", 43 | WriteFile(openVpnPackagePath, openVpnPackageKey, "root", 0755), 44 | ) 45 | } 46 | 47 | const openvpnServerConfig = `port 1194 48 | proto tcp 49 | dev tun0 50 | ca ca.crt 51 | cert {{ .Name }}.crt 52 | key {{ .Name }}.key 53 | dh dh1024.pem 54 | server {{ .Address }} {{ .Netmask }} 55 | ifconfig-pool-persist ipp.txt 56 | {{ range .Routes }} 57 | push "{{ . }}" 58 | {{ end }} 59 | keepalive 10 120 60 | comp-lzo 61 | persist-key 62 | persist-tun 63 | status openvpn-status.log 64 | verb 3 65 | client-to-client 66 | ` 67 | 68 | const openVpnPackageKey = `#!/usr/bin/env bash 69 | set -e 70 | 71 | LOGIN=$1 72 | KEYS_DIR=/etc/openvpn/easy-rsa/keys 73 | LOGIN_DIR=$KEYS_DIR/$LOGIN.tblk 74 | CONFIG_PATH=$LOGIN_DIR/$LOGIN.conf 75 | PUBLIC_IP=$(curl -s jsonip.com | grep -o 'ip":".*"' | cut -d '"' -f 3) 76 | 77 | if [[ "$PUBLIC_IP" == "" ]]; then 78 | echo "PUBLIC_IP must not be blank" 79 | exit 80 | fi 81 | CRT_PATH=$KEYS_DIR/$LOGIN.crt 82 | KEY_PATH=$KEYS_DIR/$LOGIN.key 83 | 84 | TBLK_NAME=$LOGIN.tblk 85 | TBLK_PATH=$KEYS_DIR/$TBLK_NAME.zip 86 | 87 | OPENVPN_NAME=$LOGIN.openvpn.zip 88 | OPENVPN_PATH=$KEYS_DIR/$OPENVPN_NAME 89 | 90 | if [ ! -e $CRT_PATH ]; then 91 | echo "ERROR: key not generated" 92 | exit 1 93 | fi 94 | 95 | rm -Rf $LOGIN_DIR 96 | mkdir -p $LOGIN_DIR 97 | cp -v /etc/openvpn/ca.crt $CRT_PATH $KEY_PATH $LOGIN_DIR/ 98 | 99 | echo "client 100 | dev tun 101 | proto tcp 102 | remote $PUBLIC_IP 1194 103 | resolv-retry infinite 104 | nobind 105 | persist-key 106 | persist-tun 107 | ca ca.crt 108 | cert $LOGIN.crt 109 | key $LOGIN.key 110 | ns-cert-type server 111 | comp-lzo 112 | verb 3" > $CONFIG_PATH 113 | 114 | cd $KEYS_DIR 115 | zip -r $TBLK_PATH $TBLK_NAME 116 | echo "wrote $TBLK_PATH" 117 | 118 | cd $KEYS_DIR/$TBLK_NAME 119 | zip $OPENVPN_PATH *.* 120 | echo "wrote $OPENVPN_PATH" 121 | ` 122 | 123 | const openVpnVars = ` 124 | export EASY_RSA="$(pwd)" 125 | export OPENSSL="openssl" 126 | export PKCS11TOOL="pkcs11-tool" 127 | export GREP="grep" 128 | export KEY_CONFIG=$($EASY_RSA/whichopensslcnf $EASY_RSA) 129 | export KEY_DIR="$EASY_RSA/keys" 130 | export PKCS11_MODULE_PATH="dummy" 131 | export PKCS11_PIN="dummy" 132 | export KEY_SIZE=1024 133 | export CA_EXPIRE=3650 134 | export KEY_EXPIRE=3650 135 | export KEY_COUNTRY="{{ .Country }}" 136 | export KEY_PROVINCE="{{ .Province }}" 137 | export KEY_CITY="{{ .City }}" 138 | export KEY_ORG="{{ .Org }}" 139 | export KEY_EMAIL="{{ .Email }}" 140 | export KEY_CN= 141 | export KEY_NAME= 142 | export KEY_OU= 143 | export PKCS11_MODULE_PATH=changeme 144 | export PKCS11_PIN=1234 145 | ` 146 | 147 | type OpenVpnMasquerade struct { 148 | Interface string `urknall:"required=true"` 149 | } 150 | 151 | func (*OpenVpnMasquerade) Render(r urknall.Package) { 152 | r.AddCommands("base", 153 | WriteFile("/etc/network/if-pre-up.d/iptables", ipUp, "root", 0744), 154 | Shell("IFACE={{ .Interface }} /etc/network/if-pre-up.d/iptables"), 155 | ) 156 | } 157 | 158 | const ipUp = `#!/bin/bash -e 159 | 160 | if [[ "$IFACE" == "{{ .Interface }}" ]]; then 161 | echo 1 > /proc/sys/net/ipv4/ip_forward 162 | iptables -t nat -A POSTROUTING -o {{ .Interface }} -j MASQUERADE 163 | fi 164 | ` 165 | -------------------------------------------------------------------------------- /examples/tpl_postgis.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "github.com/dynport/urknall" 4 | 5 | type PostGis struct { 6 | Version string `urknall:"required=true"` 7 | PostgresInstallDir string `urknall:"required=true"` 8 | } 9 | 10 | func (pgis *PostGis) url() string { 11 | return "http://download.osgeo.org/postgis/source/postgis-{{ .Version }}.tar.gz" 12 | } 13 | 14 | func (pgis *PostGis) Render(pkg urknall.Package) { 15 | pkg.AddCommands("packages", 16 | InstallPackages("imagemagick", "libgeos-dev", "libproj-dev", "libgdal-dev"), 17 | ) 18 | pkg.AddCommands("download", 19 | DownloadAndExtract(pgis.url(), "/opt/src/"), 20 | ) 21 | pkg.AddCommands("build", 22 | And( 23 | "cd /opt/src/postgis-{{ .Version }}", 24 | "./configure --with-pgconfig={{ .PostgresInstallDir }}/bin/pg_config --prefix={{ .InstallDir }}", 25 | "make", 26 | "make install", 27 | ), 28 | ) 29 | } 30 | 31 | func (pgis *PostGis) InstallDir() string { 32 | if pgis.Version == "" { 33 | panic("Version must be set") 34 | } 35 | return "/opt/postgis-" + pgis.Version 36 | } 37 | -------------------------------------------------------------------------------- /examples/tpl_postgres.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "github.com/dynport/urknall" 4 | 5 | type Postgres struct { 6 | Version string `urknall:"required=true"` // 9.3.4 7 | DataDir string `urknall:"default=/data/postgres"` 8 | User string `urknall:"default=postgres"` 9 | 10 | InitDb bool 11 | } 12 | 13 | func (pgres *Postgres) Render(pkg urknall.Package) { 14 | pkg.AddCommands("packages", 15 | InstallPackages("build-essential", "openssl", "libssl-dev", "flex", "zlib1g-dev", "libxslt1-dev", "libxml2-dev", "python-dev", "libreadline-dev", "bison"), 16 | ) 17 | pkg.AddCommands("download", DownloadAndExtract("{{ .Url }}", "/opt/src/")) 18 | pkg.AddCommands("user", AddUser("{{ .User }}", true)) 19 | pkg.AddCommands("build", 20 | And( 21 | "cd /opt/src/postgresql-{{ .Version }}", 22 | "./configure --prefix={{ .InstallDir }}", 23 | "make", 24 | "make install", 25 | ), 26 | ) 27 | pkg.AddCommands("upstart", 28 | WriteFile("/etc/init/postgres.conf", postgresUpstart, "root", 0644), 29 | ) 30 | if pgres.InitDb { 31 | user := pgres.User 32 | if user == "" { 33 | user = "postgres" 34 | } 35 | pkg.AddCommands("init_db", 36 | Mkdir(pgres.DataDir, "{{ .User }}", 0755), 37 | AsUser(user, "{{ .InstallDir }}/bin/initdb -D {{ .DataDir }} -E utf8 --auth-local=trust"), 38 | ) 39 | } 40 | } 41 | 42 | func (pgres *Postgres) InstallDir() string { 43 | if pgres.Version == "" { 44 | panic("Version must be set") 45 | } 46 | return "/opt/postgresql-" + pgres.Version 47 | } 48 | 49 | // some helpers for e.g. database creation 50 | type PostgresDatabase struct { 51 | Name string 52 | Owner string 53 | } 54 | 55 | func (db *PostgresDatabase) CreateCommand() string { 56 | cmd := "CREATE DATABASE " + db.Name 57 | if db.Owner != "" { 58 | cmd += " OWNER=" + db.Owner 59 | } 60 | return cmd 61 | } 62 | 63 | type PostgresUser struct { 64 | Name string 65 | Password string 66 | } 67 | 68 | func (user *PostgresUser) CreateCommand() string { 69 | cmd := "CREATE USER " + user.Name 70 | if user.Password != "" { 71 | cmd += " WITH PASSWORD '" + user.Password + "'" 72 | } 73 | return cmd 74 | } 75 | 76 | func (pgres *Postgres) CreateDatabaseCommand(db *PostgresDatabase) string { 77 | return pgres.InstallDir() + "/bin/" + `psql -U postgres -c "` + db.CreateCommand() + `"` 78 | } 79 | 80 | func (pgres *Postgres) CreateUserCommand(user *PostgresUser) string { 81 | return pgres.InstallDir() + "/bin/" + `psql -U postgres -c "` + user.CreateCommand() + `"` 82 | } 83 | 84 | const postgresUpstart = ` 85 | start on runlevel [2345] 86 | stop on runlevel [!2345] 87 | setuid {{ .User }} 88 | exec {{ .InstallDir }}/bin/postgres -D {{ .DataDir }} 89 | ` 90 | 91 | func (pgres *Postgres) Url() string { 92 | return "http://ftp.postgresql.org/pub/source/v{{ .Version }}/postgresql-{{ .Version }}.tar.gz" 93 | } 94 | -------------------------------------------------------------------------------- /examples/tpl_rabbitmq.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "github.com/dynport/urknall" 4 | 5 | type RabbitMQ struct { 6 | Version string `urknall:"required=true"` // e.g. 3.3.4 7 | } 8 | 9 | func (amqp *RabbitMQ) Render(pkg urknall.Package) { 10 | pkg.AddCommands("packages", 11 | InstallPackages("erlang-nox", "erlang-reltool", "erlang-dev"), 12 | ) 13 | pkg.AddCommands("download", 14 | DownloadAndExtract("{{ .Url }}", "/opt/"), 15 | ) 16 | pkg.AddCommands("enable_management", 17 | Shell("cd {{ .InstallDir }} && ./sbin/rabbitmq-plugins enable rabbitmq_management"), 18 | ) 19 | pkg.AddCommands("config", 20 | WriteFile("/etc/init/rabbitmq.conf", "env HOME=/root\nexec {{ .InstallDir }}/sbin/rabbitmq-server\n", "root", 0644), 21 | ) 22 | } 23 | 24 | func (amqp *RabbitMQ) InstallDir() string { 25 | if amqp.Version == "" { 26 | panic("Version must be set") 27 | } 28 | return "/opt/rabbitmq_server-" + amqp.Version 29 | } 30 | 31 | func (amqp *RabbitMQ) Url() string { 32 | return "http://www.rabbitmq.com/releases/rabbitmq-server/v{{ .Version }}/rabbitmq-server-generic-unix-{{ .Version }}.tar.gz" 33 | } 34 | -------------------------------------------------------------------------------- /examples/tpl_redis.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/dynport/urknall" 5 | "github.com/dynport/urknall/cmd" 6 | ) 7 | 8 | type Redis struct { 9 | Version string `urknall:"required=true"` // e.g. 2.8.12 10 | Autostart bool 11 | } 12 | 13 | func (redis *Redis) InstallDir() string { 14 | if redis.Version == "" { 15 | panic("Version must be set") 16 | } 17 | return "/opt/redis-" + redis.Version 18 | } 19 | 20 | func (redis *Redis) Render(pkg urknall.Package) { 21 | pkg.AddCommands("base", 22 | InstallPackages("build-essential"), 23 | Mkdir("/opt/src/", "root", 0755), 24 | DownloadAndExtract(redis.url(), "/opt/src/"), 25 | And( 26 | "cd /opt/src/redis-{{ .Version }}", 27 | "make", 28 | "PREFIX={{ .InstallDir }} make install", 29 | ), 30 | Mkdir("/data/redis", "root", 0755), 31 | ) 32 | pkg.AddTemplate("config", &RedisConfig{}) 33 | pkg.AddTemplate("upstart", &RedisUpstart{RedisDir: redis.InstallDir(), Autostart: redis.Autostart}) 34 | } 35 | 36 | func (redis *Redis) WriteConfig(config string) cmd.Command { 37 | return WriteFile("/etc/redis.conf", config, "root", 0644) 38 | } 39 | 40 | func (redis *Redis) url() string { 41 | return "http://download.redis.io/releases/redis-{{ .Version }}.tar.gz" 42 | } 43 | 44 | type RedisConfig struct { 45 | Port int `urknall:"default=6379"` 46 | Path string `urknall:"default=/etc/redis.conf"` 47 | SyslogIdent string `urknall:"default=redis"` 48 | } 49 | 50 | func (redis *RedisConfig) Render(pkg urknall.Package) { 51 | pkg.AddCommands("base", 52 | WriteFile(redis.Path, redisCfg, "root", 0644), 53 | ) 54 | } 55 | 56 | const redisCfg = `daemonize no 57 | port {{ .Port }} 58 | timeout 0 59 | tcp-keepalive 0 60 | loglevel notice 61 | syslog-enabled yes 62 | syslog-ident {{ .SyslogIdent }} 63 | databases 16 64 | save 900 1 65 | save 300 10 66 | save 60 10000 67 | stop-writes-on-bgsave-error yes 68 | rdbcompression yes 69 | rdbchecksum yes 70 | dbfilename dump.rdb 71 | dir /data/redis 72 | slave-serve-stale-data yes 73 | slave-read-only yes 74 | repl-disable-tcp-nodelay no 75 | slave-priority 100 76 | appendonly yes 77 | appendfsync everysec 78 | no-appendfsync-on-rewrite no 79 | auto-aof-rewrite-percentage 100 80 | auto-aof-rewrite-min-size 64mb 81 | lua-time-limit 5000 82 | slowlog-log-slower-than 10000 83 | slowlog-max-len 128 84 | notify-keyspace-events "" 85 | hash-max-ziplist-entries 512 86 | hash-max-ziplist-value 64 87 | list-max-ziplist-entries 512 88 | list-max-ziplist-value 64 89 | set-max-intset-entries 512 90 | zset-max-ziplist-entries 128 91 | zset-max-ziplist-value 64 92 | activerehashing yes 93 | client-output-buffer-limit normal 0 0 0 94 | client-output-buffer-limit slave 256mb 64mb 60 95 | client-output-buffer-limit pubsub 32mb 8mb 60 96 | hz 10 97 | aof-rewrite-incremental-fsync yes 98 | ` 99 | 100 | type RedisUpstart struct { 101 | Name string `urknall:"default=redis"` 102 | RedisConfig string `urknall:"default=/etc/redis.conf"` 103 | RedisDir string `urknall:"required=true"` 104 | Autostart bool 105 | } 106 | 107 | func (u *RedisUpstart) Render(r urknall.Package) { 108 | r.AddCommands("base", 109 | WriteFile("/etc/init/{{ .Name }}.conf", redisUpstart, "root", 0644), 110 | ) 111 | return 112 | } 113 | 114 | const redisUpstart = ` 115 | {{ if .Autostart }} 116 | start on (local-filesystems and net-device-up IFACE!=lo) 117 | {{ end }} 118 | pre-start script 119 | sysctl vm.overcommit_memory=1 120 | end script 121 | exec {{ .RedisDir }}/bin/redis-server {{ .RedisConfig }} 122 | respawn 123 | respawn limit 10 60 124 | ` 125 | -------------------------------------------------------------------------------- /examples/tpl_ruby.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/dynport/urknall" 7 | ) 8 | 9 | // Ruby compiles and installs ruby from source 10 | // 11 | // Ruby will be downloaded, extracted, configured, built, and installed to `/opt/ruby-{{ .Version }}`. If the `Bundle` 12 | // flag is set, bundler will be installed. 13 | type Ruby struct { 14 | Version string `urknall:"required=true"` 15 | Local bool // install to /usr/local/bin 16 | } 17 | 18 | func (ruby *Ruby) Render(pkg urknall.Package) { 19 | pkg.AddCommands("packages", 20 | InstallPackages( 21 | "curl", "build-essential", "libyaml-dev", "libxml2-dev", "libxslt1-dev", "libreadline-dev", "libssl-dev", "zlib1g-dev", 22 | ), 23 | ) 24 | pkg.AddCommands("download", DownloadAndExtract("{{ .Url }}", "/opt/src")) 25 | pkg.AddCommands("build", 26 | And( 27 | "cd {{ .SourceDir }}", 28 | "./configure --disable-install-doc --prefix={{ .InstallDir }}", 29 | "make", 30 | "make install", 31 | ), 32 | ) 33 | } 34 | 35 | func (ruby *Ruby) Url() string { 36 | return "http://ftp.ruby-lang.org/pub/ruby/{{ .MinorVersion }}/ruby-{{ .Version }}.tar.gz" 37 | } 38 | 39 | func (ruby *Ruby) MinorVersion() string { 40 | return strings.Join(strings.Split(ruby.Version, ".")[0:2], ".") 41 | } 42 | 43 | func (ruby *Ruby) InstallDir() string { 44 | if ruby.Local { 45 | return "/usr/local" 46 | } 47 | if ruby.Version == "" { 48 | panic("Version must be set") 49 | } 50 | return "/opt/ruby-" + ruby.Version 51 | } 52 | 53 | func (ruby *Ruby) SourceDir() string { 54 | return "/opt/src/ruby-{{ .Version }}" 55 | } 56 | -------------------------------------------------------------------------------- /examples/tpl_syslogng.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "github.com/dynport/urknall" 4 | 5 | const syslogNgRestart = "{ status syslog-ng | grep running && restart syslog-ng; } || start syslog-ng" 6 | 7 | type SyslogNg struct { 8 | Version string `urknall:"required=true"` 9 | } 10 | 11 | func (ng *SyslogNg) Url() string { 12 | return "http://www.balabit.com/downloads/files/syslog-ng/open-source-edition/{{ .Version }}/source/syslog-ng_{{ .Version }}.tar.gz" 13 | } 14 | 15 | func (ng *SyslogNg) Render(pkg urknall.Package) { 16 | pkg.AddCommands("packages", 17 | InstallPackages("build-essential", "libevtlog-dev", "pkg-config", "libglib2.0-dev"), 18 | ) 19 | pkg.AddCommands("download", DownloadAndExtract("{{ .Url }}", "/opt/src")) 20 | pkg.AddCommands("build", 21 | And( 22 | "cd {{ .SrcDir }}", 23 | "./configure", 24 | "make", 25 | "make install", 26 | ), 27 | ) 28 | pkg.AddCommands("upstart", WriteFile("/etc/init/syslog-ng.conf", syslogNgUpstart, "root", 0644)) 29 | } 30 | 31 | func (ng *SyslogNg) SrcDir() string { 32 | return "/opt/src/syslog-ng-{{ .Version }}" 33 | } 34 | 35 | const syslogNgUpstart = `# syslog-ng - system logging daemon 36 | # 37 | # syslog-ng is an replacement for the traditionala syslog daemon, logging messages from applications 38 | 39 | description "system logging daemon" 40 | 41 | start on filesystem 42 | stop on runlevel [06] 43 | 44 | env LD_LIBRARY_PATH=/usr/local/lib 45 | 46 | respawn 47 | 48 | exec syslog-ng -F 49 | ` 50 | 51 | type SyslogNgReceiver struct { 52 | Version string `urknall:"required=true"` // e.g. 3.5.4.1 53 | LogsRoot string `urknall:"default=/var/log/hourly"` 54 | AmqpHost string 55 | } 56 | 57 | func (ngRecv *SyslogNgReceiver) Render(pkg urknall.Package) { 58 | pkg.AddTemplate("base", &SyslogNg{Version: ngRecv.Version}) 59 | pkg.AddTemplate("symlink", &CreateHourlySymlinks{Root: ngRecv.LogsRoot}) 60 | pkg.AddCommands("config", 61 | WriteFile("/usr/local/etc/syslog-ng.conf", syslogReceiver, "root", 0644), 62 | Shell(syslogNgRestart), 63 | ) 64 | } 65 | 66 | type SyslogNgSender struct { 67 | Receiver string 68 | Version string `urknall:"required=true"` // e.g. 3.5.4.1 69 | } 70 | 71 | func (ngSend *SyslogNgSender) Render(pkg urknall.Package) { 72 | pkg.AddTemplate("base", 73 | &SyslogNg{Version: ngSend.Version}, 74 | ) 75 | pkg.AddCommands("config", 76 | WriteFile("/usr/local/etc/syslog-ng.conf", syslogNgSender, "root", 0644), 77 | Shell(syslogNgRestart), 78 | ) 79 | } 80 | 81 | const syslogNgSender = `@version: {{ .Version }} 82 | @include "scl.conf" 83 | 84 | options { 85 | chain_hostnames(0); 86 | keep_hostname(yes); 87 | time_reopen(10); 88 | time_reap(360); 89 | log_fifo_size(2048); 90 | create_dirs(yes); 91 | perm(0640); 92 | dir_perm(0755); 93 | use_dns(no); 94 | stats_freq(43200); 95 | frac_digits(6); 96 | ts_format(iso); 97 | }; 98 | 99 | source s_network { 100 | udp(port(514)); 101 | tcp(port(514)); 102 | }; 103 | 104 | source s_local { 105 | file("/proc/kmsg"); 106 | unix-stream("/dev/log"); 107 | internal(); 108 | }; 109 | 110 | destination d_syslog_tcp { 111 | syslog("{{ .Receiver }}" transport("tcp")); 112 | }; 113 | 114 | log { 115 | source(s_local); 116 | source(s_network); 117 | destination(d_syslog_tcp); 118 | }; 119 | ` 120 | 121 | type CreateHourlySymlinks struct { 122 | Root string `urknall:"default=/var/log/hourly"` 123 | } 124 | 125 | func (*CreateHourlySymlinks) Render(pkg urknall.Package) { 126 | pkg.AddCommands("base", 127 | Mkdir("/opt/scripts", "root", 0755), 128 | WriteFile("/opt/scripts/create_hourly_symlinks.sh", createHourlySymlinks, "root", 0755), 129 | WriteFile("/etc/cron.d/create_hourly_symlinks", "* * * * * root /opt/scripts/create_hourly_symlinks.sh 2>&1 | logger -i -t create_hourly_symlinks\n", "root", 0644), 130 | ) 131 | } 132 | 133 | const createHourlySymlinks = ` 134 | #!/usr/bin/env bash 135 | set -e 136 | 137 | LOG_DIR={{ .Root }} 138 | NOW=$LOG_DIR/$(date +"%Y/%m/%d/%Y-%m-%dT%H.log") 139 | TODAY=$(dirname $NOW) 140 | 141 | mkdir -p $TODAY 142 | touch $NOW 143 | chmod 0644 $NOW 144 | ln -nfs $NOW $LOG_DIR/current 145 | ln -nfs $TODAY $LOG_DIR/today 146 | ` 147 | 148 | const syslogReceiver = `@version: {{ .Version }} 149 | @include "scl.conf" 150 | 151 | options { 152 | chain_hostnames(0); 153 | keep_hostname(yes); 154 | time_reopen(10); 155 | time_reap(360); 156 | log_fifo_size(2048); 157 | create_dirs(yes); 158 | perm(0640); 159 | dir_perm(0755); 160 | use_dns(no); 161 | stats_freq(43200); 162 | frac_digits(6); 163 | ts_format(iso); 164 | }; 165 | 166 | source s_network { 167 | udp(port(514)); 168 | tcp(port(514)); 169 | }; 170 | 171 | source s_local { 172 | file("/proc/kmsg"); 173 | unix-stream("/dev/log"); 174 | internal(); 175 | }; 176 | 177 | {{ with .AmqpHost }} 178 | destination d_amqp { 179 | amqp( 180 | vhost("/") 181 | host("{{ . }}") 182 | port(5672) 183 | username("guest") # required option, no default 184 | password("guest") # required option, no default 185 | exchange("syslog") 186 | exchange_declare(yes) 187 | exchange_type("fanout") 188 | routing_key("$HOST.$PROGRAM.$PRIORITY") 189 | body("$S_ISODATE $HOST $PROGRAM.$PRIORITY[$PID]: $MSG\n") 190 | persistent(yes) 191 | frac_digits(6) 192 | value-pairs( 193 | scope("selected-macros" "nv-pairs" "sdata") 194 | ) 195 | ); 196 | }; 197 | {{ end }} 198 | 199 | destination d_file { 200 | file( 201 | "{{ .LogsRoot }}/$R_YEAR/$R_MONTH/$R_DAY/$R_YEAR-$R_MONTH-${R_DAY}T${R_HOUR}.log" 202 | template("$S_ISODATE $HOST $PROGRAM.$PRIORITY[$PID]: $MSG\n") 203 | template_escape(no) 204 | perm( 0644 ) 205 | dir_perm( 0775 ) 206 | frac_digits(6) 207 | ); 208 | }; 209 | 210 | log { 211 | source(s_local); 212 | source(s_network); 213 | {{ with .AmqpHost }}destination(d_amqp);{{ end }} 214 | destination(d_file); 215 | }; 216 | ` 217 | -------------------------------------------------------------------------------- /examples/tpl_system.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/dynport/urknall" 7 | ) 8 | 9 | type Hostname struct { 10 | Hostname string `urknall:"required=true"` 11 | } 12 | 13 | func (h *Hostname) Render(pkg urknall.Package) { 14 | pkg.AddCommands("base", 15 | Shell("hostname localhost"), // Set hostname to make sudo happy. 16 | &FileCommand{Path: "/etc/hostname", Content: h.Hostname}, 17 | &FileCommand{Path: "/etc/hosts", Content: "127.0.0.1 {{ .Hostname }} localhost"}, 18 | Shell("hostname -F /etc/hostname"), 19 | ) 20 | } 21 | 22 | type System struct { 23 | Timezone string 24 | 25 | // limits 26 | LimitsDefaults bool 27 | 28 | // sysctl 29 | SysctlDefaults bool 30 | ShmMax string 31 | ShmAll string 32 | 33 | SwapInMB int 34 | } 35 | 36 | const TimezoneUTC = "Etc/UTC" 37 | 38 | func (tpl *System) Render(pkg urknall.Package) { 39 | if tpl.Timezone != "" { 40 | pkg.AddCommands("timezone", 41 | WriteFile("/etc/timezone", tpl.Timezone, "root", 0644), // see TimezoneUTC 42 | Shell("dpkg-reconfigure --frontend noninteractive tzdata"), 43 | ) 44 | } 45 | 46 | if tpl.SysctlDefaults { 47 | pkg.AddCommands("sysctl", 48 | WriteFile("/etc/sysctl.conf", sysctlTpl, "root", 0644), 49 | Shell("sysctl -p"), 50 | ) 51 | } 52 | 53 | if tpl.LimitsDefaults { 54 | pkg.AddCommands("limits", 55 | WriteFile("/etc/security/limits.conf", limitsTpl, "root", 0644), 56 | Shell("ulimit -a"), 57 | ) 58 | } 59 | 60 | if tpl.SwapInMB > 0 { 61 | pkg.AddCommands("swap", 62 | Shell(fmt.Sprintf("swapoff -a && rm -f /swapfile && fallocate -l %dM /swapfile", tpl.SwapInMB)), 63 | Shell("chmod 0600 /swapfile"), 64 | Shell("mkswap /swapfile"), 65 | Shell("grep '/swapfile' /etc/fstab > /dev/null || echo '/swapfile none swap defaults 0 0' >> /etc/fstab"), 66 | Shell("swapon -a"), 67 | ) 68 | } 69 | } 70 | 71 | const limitsTpl = `* soft nofile 65535 72 | * hard nofile 65535 73 | root soft nofile 65535 74 | root hard nofile 65535 75 | ` 76 | 77 | const sysctlTpl = `net.core.rmem_max=16777216 78 | net.core.wmem_max=16777216 79 | net.core.wmem_default=262144 80 | net.ipv4.tcp_rmem=4096 87380 16777216 81 | net.ipv4.tcp_wmem=4096 65536 16777216 82 | net.core.netdev_max_backlog=4000 83 | net.ipv4.tcp_low_latency=1 84 | net.ipv4.tcp_window_scaling=1 85 | net.ipv4.tcp_timestamps=1 86 | net.ipv4.tcp_sack=1 87 | fs.file-max=65535 88 | net.core.wmem_default=8388608 89 | net.core.rmem_default=8388608 90 | net.core.netdev_max_backlog=10000 91 | net.core.somaxconn=4000 92 | net.ipv4.tcp_max_syn_backlog=40000 93 | net.ipv4.tcp_fin_timeout=15 94 | net.ipv4.tcp_tw_reuse=1 95 | vm.swappiness=0 96 | {{ if .ShmMax }}kernel.shmmax={{ .ShmMax }}{{ end }} 97 | {{ if .ShmAll }}kernel.shmmax={{ .ShmAll }}{{ end }} 98 | ` 99 | -------------------------------------------------------------------------------- /examples/urknall-hello-example/cmd_shell.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/dynport/urknall/utils" 8 | ) 9 | 10 | func Shell(cmd string) *ShellCommand { 11 | return &ShellCommand{Command: cmd} 12 | } 13 | 14 | type ShellCommand struct { 15 | Command string // Command to be executed in the shell. 16 | user string // User to run the command as. 17 | } 18 | 19 | func (cmd *ShellCommand) Render(i interface{}) { 20 | cmd.Command = utils.MustRenderTemplate(cmd.Command, i) 21 | } 22 | 23 | func (sc *ShellCommand) Shell() string { 24 | if sc.isExecutedAsUser() { 25 | return fmt.Sprintf("su -l %s < "text/plain"}, ["Hello World!"]] 108 | end 109 | 110 | run app 111 | ` 112 | -------------------------------------------------------------------------------- /examples/urknall-rack-example/cmd_add_user.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "fmt" 4 | 5 | func AddUser(name string, systemUser bool) *ShellCommand { 6 | testForUser := "id " + name + " 2>&1 > /dev/null" 7 | userAddOpts := "" 8 | if systemUser { 9 | userAddOpts = "--system" 10 | } else { 11 | userAddOpts = "-m -s /bin/bash" 12 | } 13 | return Or(testForUser, fmt.Sprintf("useradd %s %s", userAddOpts, name)) 14 | } 15 | -------------------------------------------------------------------------------- /examples/urknall-rack-example/cmd_as_user.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "fmt" 4 | 5 | // Convenience function to run a command as a certain user. Setting an empty user will do nothing, as the command is 6 | // then executed as "root". Note that nested calls will not work. The function will panic if it detects such a scenario. 7 | func AsUser(user string, i interface{}) *ShellCommand { 8 | switch c := i.(type) { 9 | case *ShellCommand: 10 | if c.isExecutedAsUser() { 11 | panic(`nesting "AsUser" calls not supported`) 12 | } 13 | c.user = user 14 | return c 15 | case string: 16 | return &ShellCommand{Command: c, user: user} 17 | default: 18 | panic(fmt.Sprintf(`type "%T" not supported`, c)) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /examples/urknall-rack-example/cmd_bool.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | // Combine the given commands with "and", i.e. all commands must succeed. Execution is stopped immediately if one of the 9 | // commands fails, the subsequent ones are not executed! If only one command is given nothing happens. 10 | func And(cmd interface{}, cmds ...interface{}) *ShellCommand { 11 | cs := mergeSubCommands(cmd, cmds...) 12 | 13 | finalCommand := fmt.Sprintf("{ %s; }", strings.Join(cs, " && ")) 14 | if len(cs) == 1 { 15 | finalCommand = cs[0] 16 | } 17 | return &ShellCommand{Command: finalCommand} 18 | } 19 | 20 | // Combine the given commands with "or", i.e. try one after one, untill the first returns success. If only a single 21 | // command is given, nothing happens. 22 | func Or(cmd interface{}, cmds ...interface{}) *ShellCommand { 23 | cs := mergeSubCommands(cmd, cmds...) 24 | 25 | finalCommand := fmt.Sprintf("{ %s; }", strings.Join(cs, " || ")) 26 | if len(cs) == 1 { 27 | finalCommand = cs[0] 28 | } 29 | return &ShellCommand{Command: finalCommand} 30 | } 31 | 32 | func mergeSubCommands(cmd interface{}, cmds ...interface{}) (cs []string) { 33 | cmdList := make([]interface{}, 0, len(cmds)+1) 34 | cmdList = append(cmdList, cmd) 35 | cmdList = append(cmdList, cmds...) 36 | 37 | for i := range cmdList { 38 | switch cmd := cmdList[i].(type) { 39 | case *ShellCommand: 40 | if cmd.user != "" && cmd.user != "root" { 41 | panic("AsUser not supported in nested commands") 42 | } 43 | cs = append(cs, cmd.Command) 44 | case string: 45 | if cmd == "" { // ignore empty commands 46 | panic("empty command found") 47 | } 48 | cs = append(cs, cmd) 49 | default: 50 | panic(fmt.Sprintf(`type "%T" not supported`, cmd)) 51 | } 52 | } 53 | return cs 54 | } 55 | -------------------------------------------------------------------------------- /examples/urknall-rack-example/cmd_download.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path" 7 | "strings" 8 | 9 | "github.com/dynport/urknall/utils" 10 | ) 11 | 12 | const TMP_DOWNLOAD_DIR = "/tmp/downloads" 13 | 14 | // Download the URL and write the file to the given destination, with owner and permissions set accordingly. 15 | // Destination can either be an existing directory or a file. If a directory is given the downloaded file will moved 16 | // there using the file name from the URL. If it is a file, the downloaded file will be moved (and possibly renamed) to 17 | // that destination. If the extract flag is set the downloaded file will be extracted to the directory given in the 18 | // destination field. 19 | type DownloadCommand struct { 20 | Url string // Where to download from. 21 | Destination string // Where to put the downloaded file. 22 | Owner string // Owner of the downloaded file. 23 | Permissions os.FileMode // Permissions of the downloaded file. 24 | Extract bool // Extract the downloaded archive. 25 | } 26 | 27 | // Download the file from the given URL and extract it to the given directory. If the directory does not exist it is 28 | // created. See the "ExtractFile" command for a list of supported archive types. 29 | func DownloadAndExtract(url, destination string) *DownloadCommand { 30 | return &DownloadCommand{Url: url, Destination: destination, Extract: true} 31 | } 32 | 33 | func Download(url, destination, owner string, permissions os.FileMode) *DownloadCommand { 34 | return &DownloadCommand{Url: url, Destination: destination, Owner: owner, Permissions: permissions} 35 | } 36 | 37 | func (cmd *DownloadCommand) Validate() error { 38 | if cmd.Url == "" { 39 | return fmt.Errorf("Url must be set") 40 | } 41 | if cmd.Destination == "" { 42 | return fmt.Errorf("Destination to download %q to must be set", cmd.Url) 43 | } 44 | return nil 45 | } 46 | 47 | func (cmd *DownloadCommand) Render(i interface{}) { 48 | cmd.Url = utils.MustRenderTemplate(cmd.Url, i) 49 | cmd.Destination = utils.MustRenderTemplate(cmd.Destination, i) 50 | } 51 | 52 | func (dc *DownloadCommand) Shell() string { 53 | filename := path.Base(dc.Url) 54 | destination := fmt.Sprintf("%s/%s", TMP_DOWNLOAD_DIR, filename) 55 | 56 | cmd := []string{} 57 | 58 | cmd = append(cmd, fmt.Sprintf("which curl > /dev/null || { apt-get update && apt-get install -y curl; }")) 59 | cmd = append(cmd, fmt.Sprintf("mkdir -p %s", TMP_DOWNLOAD_DIR)) 60 | cmd = append(cmd, fmt.Sprintf("cd %s", TMP_DOWNLOAD_DIR)) 61 | cmd = append(cmd, fmt.Sprintf(`curl -SsfLO "%s"`, dc.Url)) 62 | 63 | switch { 64 | case dc.Extract && dc.Destination == "": 65 | panic(fmt.Errorf("shall extract, but don't know where (i.e. destination field is empty")) 66 | case dc.Extract: 67 | cmd = append(cmd, ExtractFile(destination, dc.Destination).Shell()) 68 | case dc.Destination != "": 69 | cmd = append(cmd, fmt.Sprintf("mv %s %s", destination, dc.Destination)) 70 | destination = dc.Destination 71 | } 72 | 73 | if dc.Owner != "" && dc.Owner != "root" { 74 | ifFile := fmt.Sprintf("{ if [ -f %s ]; then chown %s %s; fi; }", destination, dc.Owner, destination) 75 | ifInDir := fmt.Sprintf("{ if [ -d %s && -f %s/%s ]; then chown %s %s/%s; fi; }", destination, destination, filename, dc.Owner, destination, filename) 76 | ifDir := fmt.Sprintf("{ if [ -d %s ]; then chown -R %s %s; fi; }", destination, dc.Owner, destination) 77 | err := `{ echo "Couldn't determine target" && exit 1; }` 78 | cmd = append(cmd, fmt.Sprintf("{ %s; }", strings.Join([]string{ifFile, ifInDir, ifDir, err}, " || "))) 79 | } 80 | 81 | if dc.Permissions != 0 { 82 | ifFile := fmt.Sprintf("{ if [ -f %s ]; then chmod %o %s; fi; }", destination, dc.Permissions, destination) 83 | ifInDir := fmt.Sprintf("{ if [ -d %s && -f %s/%s ]; then chmod %o %s/%s; fi; }", destination, destination, 84 | filename, dc.Permissions, destination, filename) 85 | ifDir := fmt.Sprintf("{ if [ -d %s ]; then chmod %o %s; fi; }", destination, dc.Permissions, destination) 86 | err := `{ echo "Couldn't determine target" && exit 1; }` 87 | cmd = append(cmd, fmt.Sprintf("{ %s; }", strings.Join([]string{ifFile, ifInDir, ifDir, err}, " || "))) 88 | } 89 | 90 | return strings.Join(cmd, " && ") 91 | } 92 | 93 | func (dc *DownloadCommand) Logging() string { 94 | sList := []string{"[DWNLOAD]"} 95 | 96 | if dc.Owner != "" && dc.Owner != "root" { 97 | sList = append(sList, fmt.Sprintf("[CHOWN:%s]", dc.Owner)) 98 | } 99 | 100 | if dc.Permissions != 0 { 101 | sList = append(sList, fmt.Sprintf("[CHMOD:%.4o]", dc.Permissions)) 102 | } 103 | 104 | sList = append(sList, fmt.Sprintf(" >> downloading file %q", dc.Url)) 105 | if dc.Extract { 106 | sList = append(sList, " and extracting archive") 107 | } 108 | if dc.Destination != "" { 109 | sList = append(sList, fmt.Sprintf(" to %q", dc.Destination)) 110 | } 111 | return strings.Join(sList, "") 112 | } 113 | -------------------------------------------------------------------------------- /examples/urknall-rack-example/cmd_extract.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "path" 6 | "strings" 7 | ) 8 | 9 | // Extract the file at the given directory. The following file extensions are currently supported (".tar", ".tgz", 10 | // ".tar.gz", ".tbz", ".tar.bz2" for tar archives, and ".zip" for zipfiles). 11 | func ExtractFile(file, targetDir string) *ShellCommand { 12 | if targetDir == "" { 13 | panic("empty target directory given") 14 | } 15 | 16 | var extractCmd *ShellCommand 17 | switch { 18 | case strings.HasSuffix(file, ".tar"): 19 | extractCmd = extractTarArchive(file, targetDir, "") 20 | case strings.HasSuffix(file, ".tgz"): 21 | fallthrough 22 | case strings.HasSuffix(file, ".tar.gz"): 23 | extractCmd = extractTarArchive(file, targetDir, "gz") 24 | case strings.HasSuffix(file, ".tbz"): 25 | fallthrough 26 | case strings.HasSuffix(file, ".tar.bz2"): 27 | extractCmd = extractTarArchive(file, targetDir, "bz2") 28 | case strings.HasSuffix(file, ".zip"): 29 | extractCmd = &ShellCommand{Command: fmt.Sprintf("unzip -d %s %s", targetDir, file)} 30 | default: 31 | panic(fmt.Sprintf("type of file %q not a supported archive", path.Base(file))) 32 | } 33 | 34 | return And( 35 | Mkdir(targetDir, "", 0), 36 | extractCmd) 37 | } 38 | 39 | func extractTarArchive(path, targetDir, compression string) *ShellCommand { 40 | additionalCommand := "" 41 | switch compression { 42 | case "gz": 43 | additionalCommand = "z" 44 | case "bz2": 45 | additionalCommand = "j" 46 | } 47 | return And( 48 | fmt.Sprintf("cd %s", targetDir), 49 | fmt.Sprintf("tar xf%s %s", additionalCommand, path)) 50 | } 51 | -------------------------------------------------------------------------------- /examples/urknall-rack-example/cmd_fileutils.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | ) 7 | 8 | // Create the given directory with the owner and file permissions set accordingly. If the last two options are set to 9 | // go's default values nothing is done. 10 | func Mkdir(path, owner string, permissions os.FileMode) *ShellCommand { 11 | if path == "" { 12 | panic("empty path given to mkdir") 13 | } 14 | 15 | mkdirCmd := fmt.Sprintf("mkdir -p %s", path) 16 | 17 | optsCmds := make([]interface{}, 0, 2) 18 | if owner != "" { 19 | optsCmds = append(optsCmds, fmt.Sprintf("chown %s %s", owner, path)) 20 | } 21 | 22 | if permissions != 0 { 23 | optsCmds = append(optsCmds, fmt.Sprintf("chmod %o %s", permissions, path)) 24 | } 25 | 26 | return And(mkdirCmd, optsCmds...) 27 | } 28 | -------------------------------------------------------------------------------- /examples/urknall-rack-example/cmd_shell.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/dynport/urknall/utils" 8 | ) 9 | 10 | func Shell(cmd string) *ShellCommand { 11 | return &ShellCommand{Command: cmd} 12 | } 13 | 14 | type ShellCommand struct { 15 | Command string // Command to be executed in the shell. 16 | user string // User to run the command as. 17 | } 18 | 19 | func (cmd *ShellCommand) Render(i interface{}) { 20 | cmd.Command = utils.MustRenderTemplate(cmd.Command, i) 21 | } 22 | 23 | func (sc *ShellCommand) Shell() string { 24 | if sc.isExecutedAsUser() { 25 | return fmt.Sprintf("su -l %s < %s", encoded, tmpPath) 80 | if fc.Owner != "" { // If owner given, change accordingly. 81 | cmd += fmt.Sprintf(" && chown %s %s", fc.Owner, tmpPath) 82 | } 83 | if fc.Permissions > 0 { // If mode given, change accordingly. 84 | cmd += fmt.Sprintf(" && chmod %o %s", fc.Permissions, tmpPath) 85 | } 86 | // Move the temporary file to the requested location. 87 | cmd += fmt.Sprintf(" && mv %s %s", tmpPath, fc.Path) 88 | return cmd 89 | } 90 | 91 | func (fc *FileCommand) Logging() string { 92 | sList := []string{"[FILE ]"} 93 | 94 | if fc.Owner != "" && fc.Owner != "root" { 95 | sList = append(sList, fmt.Sprintf("[CHOWN:%s]", fc.Owner)) 96 | } 97 | 98 | if fc.Permissions != 0 { 99 | sList = append(sList, fmt.Sprintf("[CHMOD:%.4o]", fc.Permissions)) 100 | } 101 | 102 | sList = append(sList, " "+fc.Path) 103 | 104 | cLen := len(fc.Content) 105 | if cLen > 50 { 106 | cLen = 50 107 | } 108 | //sList = append(sList, fmt.Sprintf(" << %s", strings.Replace(string(fc.Content[0:cLen]), "\n", "⁋", -1))) 109 | return strings.Join(sList, "") 110 | } 111 | -------------------------------------------------------------------------------- /examples/urknall-rack-example/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "log" 7 | "os" 8 | 9 | "github.com/dynport/urknall" 10 | ) 11 | 12 | var logger = log.New(os.Stderr, "", 0) 13 | 14 | // execute with urknall-rack-example -l -H -p 15 | func main() { 16 | if e := run(); e != nil { 17 | logger.Printf("ERROR: " + e.Error()) 18 | flag.Usage() 19 | } 20 | } 21 | 22 | func run() error { 23 | // Setup logging to stdout 24 | // all logs are published to a pubsub system and urknall.OpenLogger adds a custom consumer 25 | // which writes to the provided io.Writer (in that case os.Stdout) 26 | // the logger needs to be closed to flush pending logs 27 | defer urknall.OpenLogger(os.Stdout).Close() 28 | 29 | // get urknall.Target from flags 30 | target, e := targetFromFlags() 31 | if e != nil { 32 | return e 33 | } 34 | // Execute a urknall.Template (App) on the provided target 35 | return urknall.Run(target, &App{RubyVersion: "2.1.2", User: "app"}) 36 | } 37 | 38 | var ( 39 | host = flag.String("H", "", "SSH host (required)") 40 | login = flag.String("l", "", "SSH login") 41 | password = flag.String("p", "", "SSH password") 42 | ) 43 | 44 | func targetFromFlags() (urknall.Target, error) { 45 | flag.Parse() 46 | 47 | if *host == "" { 48 | return nil, fmt.Errorf("no host provided") 49 | } 50 | creds := *host 51 | 52 | if *login != "" { 53 | creds = *login + "@" + creds 54 | } 55 | logger.Printf("using ssh %q", creds) 56 | 57 | if *password != "" { 58 | // urknall.NewSshTargetWithPassword creates a target with password authentication 59 | // use @ to specify ssh login and host. 60 | // only use "" to use the defaults (either your local user or from $HOME/.ssh/config 61 | return urknall.NewSshTargetWithPassword(creds, *password) 62 | } 63 | 64 | // use urknall.NewSshTarget to use public key (using local ssh-agent) authentication 65 | return urknall.NewSshTarget(creds) 66 | } 67 | -------------------------------------------------------------------------------- /examples/urknall-rack-example/ruby.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/dynport/urknall" 7 | ) 8 | 9 | // Ruby is a urknall.Template to install ruby from source 10 | // 11 | // Version is a required variable which is used when "rendering" steps/commands 12 | // All public attributes and methods of a Template can be used when rendering 13 | type Ruby struct { 14 | Version string `urknall:"required=true"` 15 | } 16 | 17 | // steps taken from from https://gorails.com/setup/ubuntu/14.04 ("from source") 18 | func (tpl *Ruby) Render(p urknall.Package) { 19 | // install packages required by ruby/rails 20 | p.AddCommands("packages", InstallPackages( 21 | "git-core", "curl", "zlib1g-dev", "build-essential", "libssl-dev", 22 | "libreadline-dev", "libyaml-dev", "libsqlite3-dev", "sqlite3", 23 | "libxml2-dev", "libxslt1-dev", "libcurl4-openssl-dev", "python-software-properties", 24 | ), 25 | ) 26 | p.AddCommands("download", 27 | // create src directory 28 | Mkdir("/opt/src/", "root", 0755), 29 | 30 | // download ruby source to /opt/src/ with user=root and chmod=0644 31 | Download( // 32 | "http://ftp.ruby-lang.org/pub/ruby/{{ .MinorVersion }}/ruby-{{ .Version }}.tar.gz", 33 | "/opt/src/", 34 | "root", 644, 35 | ), 36 | ) 37 | 38 | // execute the build steps in one concatenated command (with &&) 39 | p.AddCommands("build", 40 | And( 41 | "cd /opt/src/", 42 | "tar xvfz ruby-{{ .Version }}.tar.gz", 43 | "cd ruby-{{ .Version }}", 44 | "./configure --disable-install-doc", 45 | "make -j 8", 46 | "make install", 47 | ), 48 | ) 49 | } 50 | 51 | func (r *Ruby) MinorVersion() string { 52 | parts := strings.Split(r.Version, ".") 53 | if len(parts) > 2 { 54 | return strings.Join(parts[0:2], ".") 55 | } 56 | panic("could not extract minor version from " + r.Version) 57 | } 58 | -------------------------------------------------------------------------------- /extract_file_test.go: -------------------------------------------------------------------------------- 1 | package urknall 2 | 3 | import "testing" 4 | 5 | func TestExtractWriteFile(t *testing.T) { 6 | s := "mkdir -p /etc/systemd/system \u0026\u0026 echo H4sIAAAJbogA/4yQTUvEMBCG7/kVwx68JXE/sLiQBUUP3mQX8bD0kE1HDaZpnUyKwv54ayu4SAUvwzC8D+/D7B+i51LcYHLkW/ZNNFusfBJbfMueMJmqca9IKiF13qG4emKk30ex341bKW7f0e3YEt8TGqlzIh0aZ4M++KhHDBI3LdBQ85841VPhIXGw6QWkg9k06WNq0fGIr5fqXK1gA7rCTsccAiw2Z3M4HmGabr8iJ+jsR8D8YZojSBltjWbgQLYwLxZqXqhVP9cXy+JyGCC7XsOy/Sb1WDOcTiv7z97FxDaEUjzayFhdf5g6B/Yy989XvcszsvgEAAD//wEAAP//y42HUsYBAAA= | base64 -d | gunzip \u003e /tmp/wunderscale.e125d5b80e2dfaafb2ae44f220572629b2b2e6bc7aeff2de2c6d7bb57e5de635 \u0026\u0026 chown root /tmp/wunderscale.e125d5b80e2dfaafb2ae44f220572629b2b2e6bc7aeff2de2c6d7bb57e5de635 \u0026\u0026 chmod 644 /tmp/wunderscale.e125d5b80e2dfaafb2ae44f220572629b2b2e6bc7aeff2de2c6d7bb57e5de635 \u0026\u0026 mv /tmp/wunderscale.e125d5b80e2dfaafb2ae44f220572629b2b2e6bc7aeff2de2c6d7bb57e5de635 /etc/systemd/system/redis.service" 7 | 8 | path, content, ok, err := extractWriteFile(s) 9 | if err != nil { 10 | t.Fatal(err) 11 | } else if !ok { 12 | t.Errorf("expected to extract file") 13 | } 14 | if v, ex := path, "/etc/systemd/system/redis.service"; ex != v { 15 | t.Errorf("expected path to be %q, was %q", ex, v) 16 | } 17 | et := "[Unit]\nDescription=Redis\nRequires=docker.service\nAfter=docker.service\n\n[Service]\nExecStartPre=-/usr/local/bin/docker stop redis\nExecStartPre=-/usr/local/bin/docker rm redis\nExecStartPre=/bin/bash -c \"/usr/local/bin/docker inspect redis:3.0.4 > /dev/null 2>&1 || /usr/local/bin/docker pull redis:3.0.4\"\nExecStart=/usr/local/bin/docker run --name=redis -p 172.17.42.1:6379:6379 -v /data/docker/redis:/data redis:3.0.4\n\n[Install]\nWantedBy=multi-user.target\n" 18 | if v, ex := content, et; ex != v { 19 | t.Errorf("expected content to be %q, was %q", ex, v) 20 | } 21 | _ = content 22 | } 23 | -------------------------------------------------------------------------------- /integration_test.go: -------------------------------------------------------------------------------- 1 | package urknall 2 | 3 | import ( 4 | "sort" 5 | "testing" 6 | 7 | "github.com/dynport/urknall/utils" 8 | ) 9 | 10 | type BuildHost struct { 11 | } 12 | 13 | func (b *BuildHost) Render(p Package) { 14 | p.AddTemplate("staging", &Staging{}) 15 | } 16 | 17 | type Staging struct { 18 | RubyVersion string `urknall:"default=2.1.2"` 19 | } 20 | 21 | func (s *Staging) Render(p Package) { 22 | p.AddTemplate("ruby-{{ .RubyVersion }}", &Ruby{Version: s.RubyVersion}) 23 | p.AddTemplate("es", &ElasticSearch{}) 24 | } 25 | 26 | type Ruby struct { 27 | Version string 28 | } 29 | 30 | type ElasticSearch struct { 31 | } 32 | 33 | func (e *ElasticSearch) Render(p Package) { 34 | p.AddCommands("install", &testCommand{cmd: "apt-get install elasticsearch"}) 35 | p.AddTemplate("ruby", &Ruby{}) 36 | } 37 | 38 | type testCommand struct { 39 | cmd string 40 | } 41 | 42 | func (c *testCommand) Shell() string { 43 | return c.cmd 44 | } 45 | 46 | func (c *testCommand) Logging() string { 47 | return c.cmd 48 | } 49 | 50 | func (c *testCommand) Render(i interface{}) { 51 | c.cmd = utils.MustRenderTemplate(c.cmd, i) 52 | } 53 | 54 | func (r *Ruby) Render(p Package) { 55 | t := NewTask() 56 | t.Add("apt-get update", "apt-get install ruby -v {{ .Version }}") 57 | p.AddTask("install", t) 58 | p.AddCommands("config", &testCommand{cmd: "echo {{ .Version }}"}) 59 | } 60 | 61 | func rcover(t *testing.T) { 62 | if r := recover(); r != nil { 63 | t.Fatal(r) 64 | } 65 | } 66 | 67 | func TestIntegration(t *testing.T) { 68 | bh := &BuildHost{} 69 | p, e := renderTemplate(bh) 70 | if e != nil { 71 | t.Errorf("didn't expect an error") 72 | } 73 | if p == nil { 74 | t.Errorf("didn't expect the template to be nil") 75 | } 76 | 77 | names := []string{} 78 | 79 | tasks := map[string]Task{} 80 | 81 | for _, task := range p.tasks { 82 | tasks[task.name] = task 83 | names = append(names, task.name) 84 | } 85 | sort.Strings(names) 86 | 87 | if len(names) != 5 { 88 | t.Errorf("expected 5 names, got %d", len(names)) 89 | } 90 | 91 | tt := []string{"staging.es.install", "staging.es.ruby.config", "staging.es.ruby.install", "staging.ruby-2.1.2.config", "staging.ruby-2.1.2.install"} 92 | for i := range tt { 93 | if names[i] != tt[i] { 94 | t.Errorf("expected names[%d] = %q, got %q", i, tt[i], names[i]) 95 | } 96 | } 97 | 98 | task := tasks["staging.ruby-2.1.2.config"] 99 | commands, e := task.Commands() 100 | if e != nil { 101 | t.Errorf("didn't expect an error, got %s", e) 102 | } 103 | if len(commands) != 1 { 104 | t.Errorf("expected to find 1 command, got %d", len(commands)) 105 | } 106 | if commands[0].Shell() != "echo 2.1.2" { 107 | t.Errorf("expected first command to be %q, got %q", "echo 2.1.2", commands[0].Shell()) 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /logger_example_test.go: -------------------------------------------------------------------------------- 1 | package urknall 2 | 3 | import "os" 4 | 5 | func ExampleOpenLogger() { 6 | logger := OpenLogger(os.Stdout) 7 | defer logger.Close() 8 | 9 | // short: defer OpenLogger(os.Stdout).Close() 10 | } 11 | -------------------------------------------------------------------------------- /logging.go: -------------------------------------------------------------------------------- 1 | package urknall 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/dynport/urknall/pubsub" 7 | ) 8 | 9 | func message(key string, hostname string, taskName string) (msg *pubsub.Message) { 10 | return &pubsub.Message{Key: key, StartedAt: time.Now(), Hostname: hostname, TaskName: taskName} 11 | } 12 | -------------------------------------------------------------------------------- /package.go: -------------------------------------------------------------------------------- 1 | package urknall 2 | 3 | import "github.com/dynport/urknall/cmd" 4 | 5 | // A template is used to modularize the urknall setting. Templates are rendered 6 | // into a package and during rendering tasks can be added. 7 | type Template interface { 8 | Render(pkg Package) 9 | } 10 | 11 | // The package is an interface. It provides the methods used to add tasks to a 12 | // package. The packages's AddTemplate and AddCommands methods will internally 13 | // create tasks. 14 | // 15 | // Nesting of templates provides a lot of flexibility as different 16 | // configurations can be used depending on the greater context. 17 | // 18 | // The first argument of all three Add methods is a string. These strings are 19 | // used as identifiers for the caching mechanism. They must be unique over all 20 | // tasks. For nested templates the identifiers are concatenated using ".". 21 | type Package interface { 22 | AddTemplate(string, Template) // Add another template, nested below the current one. 23 | AddCommands(string, ...cmd.Command) // Add a new task from the given commands. 24 | AddTask(string, Task) // Add the given tasks to the package with the given name. 25 | } 26 | -------------------------------------------------------------------------------- /package_impl.go: -------------------------------------------------------------------------------- 1 | package urknall 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/dynport/urknall/cmd" 8 | "github.com/dynport/urknall/utils" 9 | ) 10 | 11 | type packageImpl struct { 12 | tasks []*task 13 | taskNames map[string]struct{} 14 | reference interface{} // used for rendering 15 | cacheKeyPrefix string 16 | } 17 | 18 | func (pkg *packageImpl) AddCommands(name string, cmds ...cmd.Command) { 19 | if pkg.cacheKeyPrefix != "" { 20 | name = pkg.cacheKeyPrefix + "." + name 21 | } 22 | name = utils.MustRenderTemplate(name, pkg.reference) 23 | t := &task{name: name} 24 | for _, c := range cmds { 25 | if r, ok := c.(cmd.Renderer); ok { 26 | r.Render(pkg.reference) 27 | } 28 | t.Add(c) 29 | } 30 | pkg.addTask(t) 31 | } 32 | 33 | func (pkg *packageImpl) AddTemplate(name string, tpl Template) { 34 | if pkg.cacheKeyPrefix != "" { 35 | name = pkg.cacheKeyPrefix + "." + name 36 | } 37 | name = utils.MustRenderTemplate(name, pkg.reference) 38 | e := validateTemplate(tpl) 39 | if e != nil { 40 | panic(e) 41 | } 42 | if pkg.reference != nil { 43 | name = utils.MustRenderTemplate(name, pkg.reference) 44 | } 45 | pkg.validateTaskName(name) 46 | child := &packageImpl{cacheKeyPrefix: name, reference: tpl} 47 | tpl.Render(child) 48 | for _, task := range child.tasks { 49 | pkg.addTask(task) 50 | } 51 | } 52 | 53 | func (pkg *packageImpl) AddTask(name string, tsk Task) { 54 | if pkg.cacheKeyPrefix != "" { 55 | name = pkg.cacheKeyPrefix + "." + name 56 | } 57 | name = utils.MustRenderTemplate(name, pkg.reference) 58 | t := &task{name: name} 59 | cmds, e := tsk.Commands() 60 | if e != nil { 61 | panic(e) 62 | } 63 | for _, c := range cmds { 64 | t.Add(c) 65 | } 66 | pkg.addTask(t) 67 | } 68 | 69 | func (pkg *packageImpl) addTask(task *task) { 70 | pkg.validateTaskName(task.name) 71 | pkg.taskNames[task.name] = struct{}{} 72 | pkg.tasks = append(pkg.tasks, task) 73 | } 74 | 75 | func (pkg *packageImpl) validateTaskName(name string) { 76 | if name == "" { 77 | panic("package names must not be empty!") 78 | } 79 | 80 | if strings.Contains(name, " ") { 81 | panic(fmt.Sprintf(`package names must not contain spaces (%q does)`, name)) 82 | } 83 | 84 | if pkg.taskNames == nil { 85 | pkg.taskNames = map[string]struct{}{} 86 | } 87 | 88 | if _, ok := pkg.taskNames[name]; ok { 89 | panic(fmt.Sprintf("package with name %q exists already", name)) 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /package_impl_test.go: -------------------------------------------------------------------------------- 1 | package urknall 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestPackageChecksum(t *testing.T) { 8 | p := &packageImpl{} 9 | p.AddCommands("test1", Shell("echo 1"), Shell("echo 2")) 10 | p.AddCommands("test2", Shell("echo 2"), Shell("echo 2"), Shell("echo 3")) 11 | if len(p.tasks) != 2 { 12 | t.Errorf("tasks should be 2, was %d", len(p.tasks)) 13 | } 14 | task := p.tasks[0] 15 | if len(task.commands) != 2 { 16 | t.Errorf("commands should be 2, was %d", len(task.commands)) 17 | } 18 | if ex, v := p.tasks[0].commands[1].Checksum(), "9f8f29bb80830f069e821de502ec94200481550c208751d49bc7465815fff4f5"; ex != v { 19 | t.Errorf("expected cs to be %q, was %q", ex, v) 20 | } 21 | if ex, v := p.tasks[1].commands[0].Checksum(), "9f8f29bb80830f069e821de502ec94200481550c208751d49bc7465815fff4f5"; ex != v { 22 | t.Errorf("expected cs to be %q, was %q", ex, v) 23 | } 24 | } 25 | 26 | func TestPackageImplSingleArg(t *testing.T) { 27 | pkg := &packageImpl{} 28 | pkg.AddCommands("test", &testCommand{"this is a test"}) 29 | if len(pkg.tasks) != 1 { 30 | t.Errorf("expected %d tasks, got %d", 1, len(pkg.tasks)) 31 | } 32 | 33 | c, err := pkg.tasks[0].Commands() 34 | if err != nil { 35 | t.Errorf("didn't expect an error, got %q", err) 36 | } 37 | if c[0].Shell() != "this is a test" { 38 | t.Errorf("expected %q, got %q", "this is a test", c[0].Shell()) 39 | } 40 | 41 | pkg.AddCommands("test2", &testCommand{"testcmd"}) 42 | if len(pkg.tasks) != 2 { 43 | t.Errorf("expected %d tasks, got %d", 2, len(pkg.tasks)) 44 | } 45 | 46 | c, err = pkg.tasks[1].Commands() 47 | if err != nil { 48 | t.Errorf("didn't expect an error, got %q", err) 49 | } 50 | if c[0].Shell() != "testcmd" { 51 | t.Errorf("expected %q, got %q", "testcmd", c[0].Shell()) 52 | } 53 | } 54 | 55 | func TestPackageImplMultipleArgs(t *testing.T) { 56 | pkg := &packageImpl{} 57 | pkg.AddCommands("test", &testCommand{"echo hello"}, &testCommand{"echo world"}) 58 | tasks := pkg.tasks 59 | if len(pkg.tasks) != 1 { 60 | t.Errorf("expected %d tasks, got %d", 1, len(pkg.tasks)) 61 | } 62 | 63 | task := tasks[0] 64 | if task.name != "test" { 65 | t.Errorf("expected task name to be %q, got %q", "test", task.name) 66 | } 67 | 68 | c, err := task.Commands() 69 | if err != nil { 70 | t.Errorf("didn't expect an error, got %q", err) 71 | } 72 | if c[0].Shell() != "echo hello" { 73 | t.Errorf("expected %q, got %q", "echo hello", c[0].Shell()) 74 | } 75 | if c[1].Shell() != "echo world" { 76 | t.Errorf("expected %q, got %q", "echo world", c[1].Shell()) 77 | } 78 | 79 | pkg.AddCommands("test2", &testCommand{"echo cmd"}) 80 | tasks = pkg.tasks 81 | if len(pkg.tasks) != 2 { 82 | t.Errorf("expected %d tasks, got %d", 2, len(pkg.tasks)) 83 | } 84 | 85 | task = tasks[1] 86 | if task.name != "test2" { 87 | t.Errorf("expected task name to be %q, got %q", "test2", task.name) 88 | } 89 | c, err = task.Commands() 90 | if err != nil { 91 | t.Errorf("didn't expect an error, got %q", err) 92 | } 93 | if len(c) != 1 { 94 | t.Errorf("expected %d command, got %d", 1, len(pkg.tasks)) 95 | } 96 | if c[0].Shell() != "echo cmd" { 97 | t.Errorf("expected %q, got %q", "echo cmd", c[0].Shell()) 98 | } 99 | } 100 | 101 | type testPackage struct { 102 | Array []string `urknall:"required=true"` 103 | } 104 | 105 | func (tp *testPackage) Render(pkg Package) { 106 | for i := range tp.Array { 107 | pkg.AddCommands(tp.Array[i], Shell("echo "+tp.Array[i])) 108 | } 109 | } 110 | 111 | func TestTemplateWithStringSliceRequired(t *testing.T) { 112 | pkg := &packageImpl{} 113 | names := []string{"foo", "bar", "baz"} 114 | pkg.AddTemplate("test", &testPackage{Array: names}) 115 | if len(pkg.tasks) != 3 { 116 | t.Fatalf("expected %d tasks, got %d", 3, len(pkg.tasks)) 117 | } 118 | 119 | for i := range names { 120 | if pkg.tasks[i].name != "test."+names[i] { 121 | t.Errorf("task %d: expected task name %q, got %q", i, "test."+names[i], pkg.tasks[i].name) 122 | } 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /package_test.go: -------------------------------------------------------------------------------- 1 | package urknall 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func shouldBeError(actual interface{}, expected ...interface{}) string { 9 | if actual == nil { 10 | return "expected error, got nil" 11 | } 12 | 13 | err, actIsError := actual.(error) 14 | msg, msgIsString := expected[0].(string) 15 | 16 | if !actIsError { 17 | return fmt.Sprintf("expected something of type error\nexpected: %s\n actual: %s", msg, actual) 18 | } 19 | 20 | if !msgIsString { 21 | return fmt.Sprintf("expected value must be of type string: %s", expected[0]) 22 | } 23 | 24 | if err.Error() != msg { 25 | return fmt.Sprintf("error message did not match\nexpected: %s\n actual: %s", msg, actual) 26 | } 27 | 28 | return "" 29 | } 30 | 31 | type genericPkg struct { 32 | } 33 | 34 | func (p *genericPkg) Render(Package) { 35 | } 36 | 37 | func TestBoolValidationRequired(t *testing.T) { 38 | type pkg struct { 39 | Field bool `urknall:"required=true"` 40 | genericPkg 41 | } 42 | 43 | pi := &pkg{} 44 | err := validateTemplate(pi) 45 | if err == nil { 46 | t.Errorf("expected error, got none") 47 | } else if err.Error() != `[package:pkg][field:Field] type "bool" doesn't support "required" tag` { 48 | t.Errorf("got wrong error: %s", err) 49 | } 50 | } 51 | 52 | func TestByteFields(t *testing.T) { 53 | func() { 54 | type pkg struct { 55 | genericPkg 56 | Field []byte `urknall:"default='a test'"` 57 | } 58 | pi := &pkg{} 59 | err := validateTemplate(pi) 60 | if err != nil { 61 | t.Errorf("didn't expect an error, got %q", err) 62 | } 63 | if string(pi.Field) != "a test" { 64 | t.Errorf("expected field to be %q, got %q", "a test", pi.Field) 65 | } 66 | }() 67 | 68 | func() { 69 | type pkg struct { 70 | genericPkg 71 | Field []byte `urknall:"required=true"` 72 | } 73 | pi := &pkg{} 74 | if err := validateTemplate(pi); err == nil { 75 | t.Errorf("expected error, got none") 76 | } 77 | pi.Field = []byte("hello world") 78 | err := validateTemplate(pi) 79 | if err != nil { 80 | t.Errorf("didn't expect an error, got %q", err) 81 | } 82 | }() 83 | } 84 | 85 | func TestBoolValidationDefault(t *testing.T) { 86 | type pkg struct { 87 | genericPkg 88 | Field bool `urknall:"default=false"` 89 | } 90 | 91 | func() { 92 | pi := &pkg{} 93 | err := validateTemplate(pi) 94 | if err != nil { 95 | t.Errorf("didn't expect an error, got %q", err) 96 | } 97 | if pi.Field != false { 98 | t.Errorf("expected field to be %t, got %t", false, pi.Field) 99 | } 100 | }() 101 | 102 | func() { 103 | pi := &pkg{Field: false} 104 | err := validateTemplate(pi) 105 | if err != nil { 106 | t.Errorf("didn't expect an error, got %q", err) 107 | } 108 | if pi.Field != false { 109 | t.Errorf("expected field to be %t, got %t", false, pi.Field) 110 | } 111 | }() 112 | 113 | func() { 114 | pi := &pkg{Field: true} 115 | err := validateTemplate(pi) 116 | if err != nil { 117 | t.Errorf("didn't expect an error, got %q", err) 118 | } 119 | if pi.Field != true { 120 | t.Errorf("expected field to be %t, got %t", true, pi.Field) 121 | } 122 | }() 123 | } 124 | 125 | func TestBoolValidationSize(t *testing.T) { 126 | type pkg struct { 127 | genericPkg 128 | Field bool `urknall:"size=3"` 129 | } 130 | pi := &pkg{Field: true} 131 | err := validateTemplate(pi) 132 | 133 | if err == nil { 134 | t.Errorf("expected error, got none") 135 | } else if err.Error() != `[package:pkg][field:Field] type "bool" doesn't support "size" tag` { 136 | t.Errorf("got wrong error: %s", err) 137 | } 138 | } 139 | 140 | func TestBoolValidationMin(t *testing.T) { 141 | type pkg struct { 142 | genericPkg 143 | Field bool `urknall:"min=3"` 144 | } 145 | pi := &pkg{Field: true} 146 | err := validateTemplate(pi) 147 | 148 | if err == nil { 149 | t.Errorf("expected error, got none") 150 | } else if err.Error() != `[package:pkg][field:Field] type "bool" doesn't support "min" tag` { 151 | t.Errorf("got wrong error: %s", err) 152 | } 153 | } 154 | 155 | func TestBoolValidationMax(t *testing.T) { 156 | type pkg struct { 157 | genericPkg 158 | Field bool `urknall:"max=3"` 159 | } 160 | pi := &pkg{Field: true} 161 | err := validateTemplate(pi) 162 | 163 | if err == nil { 164 | t.Errorf("expected error, got none") 165 | } else if err.Error() != `[package:pkg][field:Field] type "bool" doesn't support "max" tag` { 166 | t.Errorf("got wrong error: %s", err) 167 | } 168 | } 169 | 170 | func TestIntValidationRequired(t *testing.T) { 171 | type pkg struct { 172 | genericPkg 173 | Field int `urknall:"required=true"` 174 | } 175 | 176 | pi := &pkg{} 177 | err := validateTemplate(pi) 178 | 179 | if err == nil { 180 | t.Errorf("expected error, got none") 181 | } else if err.Error() != `[package:pkg][field:Field] type "int" doesn't support "required" tag` { 182 | t.Errorf("got wrong error: %s", err) 183 | } 184 | } 185 | 186 | func TestIntValidationDefault(t *testing.T) { 187 | func() { 188 | type pkg struct { 189 | genericPkg 190 | Field int `urknall:"default=five"` 191 | } 192 | 193 | pi := &pkg{Field: 1} 194 | err := validateTemplate(pi) 195 | 196 | if err == nil { 197 | t.Errorf("expected error, got none") 198 | } else if err.Error() != `[package:pkg][field:Field] failed to parse value (not an int) of tag "default": "five"` { 199 | t.Errorf("got wrong error: %s", err) 200 | } 201 | }() 202 | 203 | func() { 204 | type pkg struct { 205 | genericPkg 206 | Field int `urknall:"default=5"` 207 | } 208 | 209 | func() { 210 | pi := &pkg{} 211 | validateTemplate(pi) 212 | if pi.Field != 5 { 213 | t.Errorf("expected field to be %d, got %d", 5, pi.Field) 214 | } 215 | }() 216 | 217 | func() { 218 | pi := &pkg{Field: 0} 219 | validateTemplate(pi) 220 | if pi.Field != 5 { 221 | t.Errorf("expected field to be %d, got %d", 5, pi.Field) 222 | } 223 | }() 224 | 225 | func() { 226 | pi := &pkg{Field: 42} 227 | validateTemplate(pi) 228 | if pi.Field != 42 { 229 | t.Errorf("expected field to be %d, got %d", 42, pi.Field) 230 | } 231 | }() 232 | }() 233 | } 234 | 235 | func TestIntValidationMin(t *testing.T) { 236 | type pkg struct { 237 | genericPkg 238 | Field int `urknall:"min=5"` 239 | } 240 | 241 | pi := &pkg{} 242 | err := validateTemplate(pi) 243 | 244 | if err == nil { 245 | t.Errorf("expected error, got none") 246 | } else if err.Error() != `[package:pkg][field:Field] value "0" smaller than the specified minimum "5"` { 247 | t.Errorf("got wrong error: %s", err) 248 | } 249 | 250 | pi.Field = 4 251 | err = validateTemplate(pi) 252 | if err == nil { 253 | t.Errorf("expected error, got none") 254 | } else if err.Error() != `[package:pkg][field:Field] value "4" smaller than the specified minimum "5"` { 255 | t.Errorf("got wrong error: %s", err) 256 | } 257 | 258 | pi.Field = 5 259 | err = validateTemplate(pi) 260 | if err != nil { 261 | t.Errorf("didn't expect an error, got %q", err) 262 | } 263 | if pi.Field != 5 { 264 | t.Errorf("expected field to be %d, got %d", 5, pi.Field) 265 | } 266 | 267 | pi.Field = 6 268 | err = validateTemplate(pi) 269 | if err != nil { 270 | t.Errorf("didn't expect an error, got %q", err) 271 | } 272 | if pi.Field != 6 { 273 | t.Errorf("expected field to be %d, got %d", 6, pi.Field) 274 | } 275 | } 276 | 277 | func TestIntValidationMax(t *testing.T) { 278 | type pkg struct { 279 | genericPkg 280 | Field int `urknall:"max=5"` 281 | } 282 | 283 | pi := &pkg{} 284 | err := validateTemplate(pi) 285 | if err != nil { 286 | t.Errorf("didn't expect an error, got %q", err) 287 | } 288 | 289 | pi.Field = 4 290 | err = validateTemplate(pi) 291 | if err != nil { 292 | t.Errorf("didn't expect an error, got %q", err) 293 | } 294 | if pi.Field != 4 { 295 | t.Errorf("expected field to be %d, got %d", 4, pi.Field) 296 | } 297 | err = validateTemplate(pi) 298 | 299 | pi.Field = 5 300 | err = validateTemplate(pi) 301 | if err != nil { 302 | t.Errorf("didn't expect an error, got %q", err) 303 | } 304 | if pi.Field != 5 { 305 | t.Errorf("expected field to be %d, got %d", 5, pi.Field) 306 | } 307 | 308 | pi.Field = 6 309 | err = validateTemplate(pi) 310 | if err == nil { 311 | t.Errorf("expected error, got none") 312 | } else if err.Error() != `[package:pkg][field:Field] value "6" greater than the specified maximum "5"` { 313 | t.Errorf("got wrong error: %s", err) 314 | } 315 | 316 | } 317 | 318 | func TestIntValidationSize(t *testing.T) { 319 | type pkg struct { 320 | genericPkg 321 | Field int `urknall:"size=5"` 322 | } 323 | 324 | pi := &pkg{} 325 | err := validateTemplate(pi) 326 | if err == nil { 327 | t.Errorf("expected error, got none") 328 | } else if err.Error() != `[package:pkg][field:Field] type "int" doesn't support "size" tag` { 329 | t.Errorf("got wrong error: %s", err) 330 | } 331 | } 332 | 333 | func TestStringValidationRequired(t *testing.T) { 334 | func() { 335 | type pkg struct { 336 | genericPkg 337 | Field string `urknall:"required=tru"` 338 | } 339 | pi := &pkg{} 340 | err := validateTemplate(pi) 341 | if err == nil { 342 | t.Errorf("expected error, got none") 343 | } else if err.Error() != `[package:pkg][field:Field] failed to parse value (neither "true" nor "false") of tag "required": "tru"` { 344 | t.Errorf("got wrong error: %s", err) 345 | } 346 | }() 347 | 348 | func() { 349 | type pkg struct { 350 | genericPkg 351 | Field string `urknall:"required=true"` 352 | } 353 | pi := &pkg{} 354 | err := validateTemplate(pi) 355 | if err == nil { 356 | t.Errorf("expected error, got none") 357 | } else if err.Error() != "[package:pkg][field:Field] required field not set" { 358 | t.Errorf("got wrong error: %s", err) 359 | } 360 | 361 | pi = &pkg{Field: ""} 362 | err = validateTemplate(pi) 363 | if err == nil { 364 | t.Errorf("expected error, got none") 365 | } else if err.Error() != "[package:pkg][field:Field] required field not set" { 366 | t.Errorf("got wrong error: %s", err) 367 | } 368 | 369 | pi = &pkg{Field: "something"} 370 | err = validateTemplate(pi) 371 | if err != nil { 372 | t.Errorf("didn't expect an error, got %q", err) 373 | } 374 | }() 375 | } 376 | 377 | func TestStringValidationDefault(t *testing.T) { 378 | type pkg struct { 379 | genericPkg 380 | Field string `urknall:"default='the 'default' value'"` 381 | } 382 | 383 | pi := &pkg{} 384 | err := validateTemplate(pi) 385 | if err != nil { 386 | t.Errorf("didn't expect an error, got %q", err) 387 | } else if pi.Field != "the 'default' value" { 388 | t.Errorf("did expect field to be %q, got %q", "the 'default' value", pi.Field) 389 | } 390 | 391 | pi = &pkg{Field: ""} 392 | err = validateTemplate(pi) 393 | if err != nil { 394 | t.Errorf("didn't expect an error, got %q", err) 395 | } else if pi.Field != "the 'default' value" { 396 | t.Errorf("did expect field to be %q, got %q", "the 'default' value", pi.Field) 397 | } 398 | 399 | pi = &pkg{Field: "some other value"} 400 | err = validateTemplate(pi) 401 | if err != nil { 402 | t.Errorf("didn't expect an error, got %q", err) 403 | } else if pi.Field != "some other value" { 404 | t.Errorf("did expect field to be %q, got %q", "some other value", pi.Field) 405 | } 406 | } 407 | 408 | func TestStringValidationMinMax(t *testing.T) { 409 | type pkg struct { 410 | genericPkg 411 | Field string `urknall:"min=3 max=4"` 412 | } 413 | pi := &pkg{} 414 | err := validateTemplate(pi) 415 | if err != nil { 416 | t.Errorf("didn't expect an error, got %q", err) 417 | } 418 | 419 | pi = &pkg{Field: "ab"} 420 | err = validateTemplate(pi) 421 | if err == nil { 422 | t.Errorf("expected error, got none") 423 | } else if err.Error() != `[package:pkg][field:Field] length of value "ab" smaller than the specified minimum length "3"` { 424 | t.Errorf("got wrong error: %s", err) 425 | } 426 | 427 | pi = &pkg{Field: "abc"} 428 | err = validateTemplate(pi) 429 | if err != nil { 430 | t.Errorf("didn't expect an error, got %q", err) 431 | } 432 | 433 | pi = &pkg{Field: "abcd"} 434 | err = validateTemplate(pi) 435 | if err != nil { 436 | t.Errorf("didn't expect an error, got %q", err) 437 | } 438 | 439 | pi = &pkg{Field: "abcde"} 440 | err = validateTemplate(pi) 441 | if err == nil { 442 | t.Errorf("expected error, got none") 443 | } else if err.Error() != `[package:pkg][field:Field] length of value "abcde" greater than the specified maximum length "4"` { 444 | t.Errorf("got wrong error: %s", err) 445 | } 446 | } 447 | 448 | func TestStringValidationSize(t *testing.T) { 449 | type pkg struct { 450 | genericPkg 451 | Field string `urknall:"size=3"` 452 | } 453 | 454 | pi := &pkg{} 455 | err := validateTemplate(pi) 456 | if err != nil { 457 | t.Errorf("didn't expect an error, got %q", err) 458 | } 459 | 460 | pi = &pkg{Field: "ab"} 461 | err = validateTemplate(pi) 462 | if err == nil { 463 | t.Errorf("expected error, got none") 464 | } else if err.Error() != `[package:pkg][field:Field] length of value "ab" doesn't match the specified size "3"` { 465 | t.Errorf("got wrong error: %s", err) 466 | } 467 | 468 | pi = &pkg{Field: "abc"} 469 | err = validateTemplate(pi) 470 | if err != nil { 471 | t.Errorf("didn't expect an error, got %q", err) 472 | } 473 | 474 | pi = &pkg{Field: "abcd"} 475 | err = validateTemplate(pi) 476 | if err == nil { 477 | t.Errorf("expected error, got none") 478 | } else if err.Error() != `[package:pkg][field:Field] length of value "abcd" doesn't match the specified size "3"` { 479 | t.Errorf("got wrong error: %s", err) 480 | } 481 | } 482 | 483 | func TestValidationRequiredInvalid(t *testing.T) { 484 | type pkg struct { 485 | genericPkg 486 | Field string `urknall:"required=aberja"` 487 | } 488 | pi := &pkg{} 489 | err := validateTemplate(pi) 490 | if err == nil { 491 | t.Errorf("expected error, got none") 492 | } else if err.Error() != `[package:pkg][field:Field] failed to parse value (neither "true" nor "false") of tag "required": "aberja"` { 493 | t.Errorf("got wrong error: %s", err) 494 | } 495 | } 496 | 497 | func TestValidationMinInvalid(t *testing.T) { 498 | type pkg struct { 499 | genericPkg 500 | Field string `urknall:"min=..3"` 501 | } 502 | pi := &pkg{} 503 | err := validateTemplate(pi) 504 | if err == nil { 505 | t.Errorf("expected error, got none") 506 | } else if err.Error() != `[package:pkg][field:Field] failed to parse value (not an int) of tag "min": "..3"` { 507 | t.Errorf("got wrong error: %s", err) 508 | } 509 | } 510 | 511 | func TestValidationMaxInvalid(t *testing.T) { 512 | type pkg struct { 513 | genericPkg 514 | Field string `urknall:"max=4a"` 515 | } 516 | pi := &pkg{} 517 | err := validateTemplate(pi) 518 | if err == nil { 519 | t.Errorf("expected error, got none") 520 | } else if err.Error() != `[package:pkg][field:Field] failed to parse value (not an int) of tag "max": "4a"` { 521 | t.Errorf("got wrong error: %s", err) 522 | } 523 | } 524 | 525 | func TestValidationSizeInvalid(t *testing.T) { 526 | type pkg struct { 527 | genericPkg 528 | Field string `urknall:"size=4a"` 529 | } 530 | pi := &pkg{} 531 | err := validateTemplate(pi) 532 | if err == nil { 533 | t.Errorf("expected error, got none") 534 | } else if err.Error() != `[package:pkg][field:Field] failed to parse value (not an int) of tag "size": "4a"` { 535 | t.Errorf("got wrong error: %s", err) 536 | } 537 | } 538 | 539 | func TestMultiTags(t *testing.T) { 540 | type pkg struct { 541 | genericPkg 542 | Field string `urknall:"default='foo' min=3 max=4"` 543 | } 544 | 545 | pi := &pkg{} 546 | err := validateTemplate(pi) 547 | if err != nil { 548 | t.Errorf("didn't expect an error, got %q", err) 549 | } 550 | if pi.Field != "foo" { 551 | t.Errorf("expect field to be set to %q, got %q", "foo", pi.Field) 552 | } 553 | 554 | pi = &pkg{Field: "ab"} 555 | err = validateTemplate(pi) 556 | if err == nil { 557 | t.Errorf("expected error, got none") 558 | } else if err.Error() != `[package:pkg][field:Field] length of value "ab" smaller than the specified minimum length "3"` { 559 | t.Errorf("got wrong error: %s", err) 560 | } 561 | } 562 | 563 | func TestTagParsing(t *testing.T) { 564 | func() { 565 | type pkg struct { 566 | genericPkg 567 | Field string `urknall:"required='abc"` 568 | } 569 | 570 | pi := &pkg{Field: "asd"} 571 | err := validateTemplate(pi) 572 | if err == nil { 573 | t.Errorf("expected error, got none") 574 | } else if err.Error() != `[package:pkg][field:Field] failed to parse tag due to erroneous quotes` { 575 | t.Errorf("got wrong error: %s", err) 576 | } 577 | }() 578 | 579 | func() { 580 | type pkg struct { 581 | genericPkg 582 | Field string `urknall:"required='ab'c'"` 583 | } 584 | 585 | pi := &pkg{Field: "asd"} 586 | err := validateTemplate(pi) 587 | if err == nil { 588 | t.Errorf("expected error, got none") 589 | } else if err.Error() != `[package:pkg][field:Field] failed to parse tag due to erroneous quotes` { 590 | t.Errorf("got wrong error: %s", err) 591 | } 592 | }() 593 | 594 | func() { 595 | type pkg struct { 596 | genericPkg 597 | Field string `urknall:"default"` 598 | } 599 | 600 | pi := &pkg{Field: "asd"} 601 | err := validateTemplate(pi) 602 | if err == nil { 603 | t.Errorf("expected error, got none") 604 | } else if err.Error() != `[package:pkg][field:Field] failed to parse annotation (value missing): "default"` { 605 | t.Errorf("got wrong error: %s", err) 606 | } 607 | }() 608 | 609 | func() { 610 | type pkg struct { 611 | genericPkg 612 | Field string `urknall:"defaul='asdf'"` 613 | } 614 | 615 | pi := &pkg{Field: "asd"} 616 | err := validateTemplate(pi) 617 | if err == nil { 618 | t.Errorf("expected error, got none") 619 | } else if err.Error() != `[package:pkg][field:Field] tag "defaul" unknown` { 620 | t.Errorf("got wrong error: %s", err) 621 | } 622 | }() 623 | } 624 | -------------------------------------------------------------------------------- /pubsub/pubsub.go: -------------------------------------------------------------------------------- 1 | package pubsub 2 | 3 | import ( 4 | "runtime" 5 | "sync" 6 | "time" 7 | 8 | "github.com/dynport/dgtk/pubsub" 9 | ) 10 | 11 | var pubSub = []*pubsub.PubSub{} 12 | var mutex = &sync.Mutex{} 13 | 14 | // Register your own instance of the PubSub type to handle logging yourself. 15 | func RegisterPubSub(ps *pubsub.PubSub) { 16 | mutex.Lock() 17 | pubSub = append(pubSub, ps) 18 | mutex.Unlock() 19 | } 20 | 21 | func publish(i interface{}) (e error) { 22 | for _, ps := range pubSub { 23 | if e = ps.Publish(i); e != nil { 24 | return e 25 | } 26 | } 27 | return nil 28 | } 29 | 30 | const ( 31 | StatusCached = "CACHED" 32 | StatusExecStart = "EXEC" 33 | StatusExecFinished = "FINISHED" 34 | ) 35 | 36 | const ( 37 | MessageTasksPrecompile = "urknall.tasks.precompile" 38 | MessageUrknallInternal = "urknall.internal" 39 | MessageCleanupCacheEntries = "urknall.cleanup_cache_entries" 40 | MessageTasksProvision = "urknall.tasks.provision.list" 41 | MessageTasksProvisionTask = "urknall.tasks.provision.task" 42 | ) 43 | 44 | // Urknall uses the http://github.com/dynport/dgtk/pubsub package for logging (a publisher-subscriber pattern where 45 | // defined messages are sent to subscribers). This is the message type urknall will send out. If you handle logging 46 | // yourself this type provides the required information. Please note that this message is sent in different context's 47 | // and not all fields will be set all the time. 48 | type Message struct { 49 | Key string // Key the message is sent with. 50 | 51 | ExecStatus string // Urknall status (executed or cached). 52 | Message string // The message to be logged. 53 | 54 | Hostname string // IP of the host a command is run. 55 | 56 | TaskName string // Name of the package currently being executed. 57 | TaskChecksum string // Hash of a task. 58 | 59 | PublishedAt time.Time // When was the message published. 60 | StartedAt time.Time // When was the message created. 61 | Duration time.Duration // How long did the action take (delta from message creation and publishing). 62 | TotalRuntime time.Duration // Timeframe of the action (might be larger than the message's). 63 | 64 | Stream string // Stream a line appeared on. 65 | Line string // Line that appeared on a stream. 66 | 67 | InvalidatedCacheEntries []string // List of invalidated cache entries (urknall caching). 68 | 69 | Error error // Error that occured. 70 | Stack string // The stack trace in case of a panic. 71 | } 72 | 73 | // Predicated to verify whether the given message was sent via stderr. 74 | func (message *Message) IsStderr() bool { 75 | return message.Stream == "stderr" 76 | } 77 | 78 | // Publish the message with the given error. 79 | func (message Message) PublishError(e error) { 80 | message.Error = e 81 | message.Publish("error") 82 | } 83 | 84 | // Publish the message with the given error and create a stacktrace. 85 | func (message *Message) PublishPanic(e error) { 86 | var buf []byte 87 | for read, size := 1024, 1024; read == size; read = runtime.Stack(buf, false) { 88 | buf = make([]byte, 2*size) 89 | } 90 | 91 | message.Stack = string(buf) 92 | message.Error = e 93 | message.Publish("panic") 94 | } 95 | 96 | // Publish the message with the given key postfix. 97 | func (message Message) Publish(key string) { 98 | if message.Key == "" { 99 | panic("message key must be set") 100 | } 101 | message.Key += ("." + key) 102 | message.PublishedAt = time.Now() 103 | if message.StartedAt.IsZero() { 104 | message.StartedAt = message.PublishedAt 105 | } else { 106 | message.Duration = message.PublishedAt.Sub(message.StartedAt) 107 | } 108 | 109 | publish(&message) 110 | } 111 | -------------------------------------------------------------------------------- /pubsub/stdout_logger.go: -------------------------------------------------------------------------------- 1 | package pubsub 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io" 7 | "strings" 8 | "time" 9 | 10 | "github.com/dynport/dgtk/pubsub" 11 | ) 12 | 13 | const ( 14 | colorDryRun = 226 15 | colorCached = 33 16 | colorExec = 34 17 | ) 18 | 19 | var colorMapping = map[string]int{ 20 | StatusCached: colorCached, 21 | StatusExecFinished: colorExec, 22 | } 23 | 24 | var ignoredMessagesError = errors.New("ignored published messages (subscriber buffer full)") 25 | 26 | // Create a logging facility for urknall using urknall's default formatter. 27 | // Note that this resource must be closed afterwards! 28 | func OpenLogger(w io.Writer) io.Closer { 29 | logger := &logger{} 30 | logger.Output = w 31 | logger.Formatter = logger.DefaultFormatter 32 | // Ignore the error from Start. It would only be triggered if the formatter wouldn't be set. 33 | _ = logger.Start() 34 | return logger 35 | } 36 | 37 | type logger struct { 38 | Output io.Writer 39 | Formatter formatter 40 | maxLengths map[int]int 41 | started time.Time 42 | finished chan interface{} 43 | pubSub *pubsub.PubSub 44 | subscription *pubsub.Subscription 45 | } 46 | 47 | func (logger *logger) Started() time.Time { 48 | if logger.started.IsZero() { 49 | logger.started = time.Now() 50 | } 51 | return logger.started 52 | } 53 | 54 | func (logger *logger) formatCommandOuput(message *Message) string { 55 | prefix := fmt.Sprintf("[%s][%s][%s]", formatIp(message.Hostname), formatTaskName(message.TaskName, 12), formatDuration(logger.sinceStarted())) 56 | line := message.Line 57 | if message.IsStderr() { 58 | line = colorize(1, line) 59 | } 60 | return prefix + " " + line 61 | } 62 | 63 | func formatIp(ip string) string { 64 | return fmt.Sprintf("%15s", ip) 65 | } 66 | 67 | type formatter func(urknallMessage *Message) string 68 | 69 | func (logger *logger) DefaultFormatter(message *Message) string { 70 | ignoreKeys := []string{MessageTasksPrecompile, MessageCleanupCacheEntries, MessageTasksProvision, MessageUrknallInternal} 71 | for _, k := range ignoreKeys { 72 | if strings.HasPrefix(message.Key, k) { 73 | return "" 74 | } 75 | } 76 | if len(message.Line) > 0 { 77 | return logger.formatCommandOuput(message) 78 | } 79 | ip := message.Hostname 80 | taskName := message.TaskName 81 | payload := "" 82 | if message.Message != "" { 83 | payload = message.Message 84 | } 85 | execStatus := fmt.Sprintf("%-8s", message.ExecStatus) 86 | if color := colorMapping[message.ExecStatus]; color > 0 { 87 | execStatus = colorize(color, execStatus) 88 | } 89 | parts := []string{ 90 | fmt.Sprintf("[%s][%s][%s][%s]%s", 91 | formatIp(ip), 92 | formatTaskName(taskName, 12), 93 | formatDuration(logger.sinceStarted()), 94 | execStatus, 95 | payload, 96 | ), 97 | } 98 | return strings.Join(parts, " ") 99 | } 100 | 101 | func formatTaskName(name string, maxLen int) string { 102 | if len(name) > maxLen { 103 | name = name[0:maxLen] 104 | } 105 | return fmt.Sprintf("%-*s", maxLen, name) 106 | } 107 | 108 | func formatDuration(dur time.Duration) string { 109 | durString := "" 110 | if dur >= 1*time.Millisecond { 111 | durString = fmt.Sprintf("%.03f", dur.Seconds()) 112 | } 113 | return fmt.Sprintf("%7s", durString) 114 | } 115 | 116 | func (logger *logger) sinceStarted() time.Duration { 117 | return time.Now().Sub(logger.Started()) 118 | } 119 | 120 | func (logger *logger) Start() error { 121 | logger.started = time.Now() 122 | if logger.Formatter == nil { 123 | return fmt.Errorf("Formatter must be set") 124 | } 125 | logger.pubSub = pubsub.New() 126 | RegisterPubSub(logger.pubSub) 127 | logger.subscription = logger.pubSub.Subscribe(func(m *Message) { 128 | if message := logger.Formatter(m); message != "" { 129 | fmt.Fprintln(logger.Output, message) 130 | } 131 | }) 132 | return nil 133 | } 134 | 135 | func (logger *logger) Close() (e error) { 136 | e = logger.subscription.Close() 137 | if d := logger.pubSub.Stats.Ignored(); e == nil && d > 0 { 138 | return ignoredMessagesError 139 | } 140 | return e 141 | } 142 | 143 | func colorize(c int, s string) string { 144 | return fmt.Sprintf("\033[38;5;%dm%s\033[0m", c, s) 145 | } 146 | -------------------------------------------------------------------------------- /runlist_test.go: -------------------------------------------------------------------------------- 1 | package urknall 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | type customCommand struct { 8 | Content string 9 | } 10 | 11 | func (cc *customCommand) Shell() string { 12 | return "cc: " + cc.Content 13 | } 14 | func (cc *customCommand) Logging() string { 15 | return "" 16 | } 17 | 18 | type somePackage struct { 19 | SField string 20 | IField int 21 | } 22 | 23 | func (sp *somePackage) Render(Package) { 24 | } 25 | 26 | func TestAddCommand(t *testing.T) { 27 | rl := &task{taskBuilder: &somePackage{SField: "something", IField: 1}} 28 | 29 | rl.Add(`string with "{{ .SField }}" and "{{ .IField }}"`) 30 | 31 | c := rl.commands[len(rl.commands)-1].command 32 | if sc, ok := c.(*stringCommand); !ok { 33 | t.Errorf("expect ok, wasn't") 34 | } else if sc.cmd != `string with "something" and "1"` { 35 | t.Errorf("expect %q, got %q", `string with "something" and "1"`, sc.cmd) 36 | } 37 | } 38 | 39 | func TestAddStringCommand(t *testing.T) { 40 | rl := &task{taskBuilder: &somePackage{SField: "something", IField: 1}} 41 | 42 | baseCommand := stringCommand{cmd: `string with "{{ .SField }}" and "{{ .IField }}"`} 43 | 44 | func() { 45 | defer func() { 46 | r := recover() 47 | if r == nil { 48 | t.Errorf("expected a panic, got none!") 49 | } 50 | }() 51 | rl.Add(baseCommand) 52 | 53 | }() 54 | 55 | rl.Add(&baseCommand) 56 | c := rl.commands[len(rl.commands)-1].command 57 | if sc, ok := c.(*stringCommand); !ok { 58 | t.Errorf("expect ok, wasn't") 59 | } else if sc.cmd != `string with "something" and "1"` { 60 | t.Errorf("expect %q, got %q", `string with "something" and "1"`, sc.cmd) 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /string_command.go: -------------------------------------------------------------------------------- 1 | package urknall 2 | 3 | import "github.com/dynport/urknall/utils" 4 | 5 | type stringCommand struct { 6 | cmd string 7 | } 8 | 9 | func (sc *stringCommand) Shell() string { 10 | return sc.cmd 11 | } 12 | 13 | func (sc *stringCommand) Logging() string { 14 | return "[COMMAND] " + sc.cmd 15 | } 16 | 17 | func (sc *stringCommand) Render(i interface{}) { 18 | sc.cmd = utils.MustRenderTemplate(sc.cmd, i) 19 | } 20 | -------------------------------------------------------------------------------- /target.go: -------------------------------------------------------------------------------- 1 | package urknall 2 | 3 | import "github.com/dynport/urknall/target" 4 | 5 | // The target interface is used to describe something a package can be built 6 | // on. 7 | type Target interface { 8 | Command(cmd string) (target.ExecCommand, error) 9 | User() string 10 | String() string 11 | Reset() error 12 | } 13 | 14 | // Create an SSH target. The address is an identifier of the form 15 | // `[@?][:port]`. It is assumed that authentication via public key 16 | // will work, i.e. the remote host has the building user's public key in its 17 | // authorized_keys file. 18 | func NewSshTarget(address string) (Target, error) { 19 | return target.NewSshTarget(address) 20 | } 21 | 22 | // Create a SSH target with a private access key 23 | func NewSshTargetWithPrivateKey(address string, key []byte) (Target, error) { 24 | return target.NewSshTargetWithPrivateKey(address, key) 25 | } 26 | 27 | // Special SSH target that uses the given password for accessing the machine. 28 | // This is required mostly for testing and shouldn't be used in production 29 | // settings. 30 | func NewSshTargetWithPassword(address, password string) (Target, error) { 31 | target, e := target.NewSshTarget(address) 32 | if e == nil { 33 | target.Password = password 34 | } 35 | return target, e 36 | } 37 | 38 | // Use the local host for building. 39 | func NewLocalTarget() (Target, error) { 40 | return target.NewLocalTarget(), nil 41 | } 42 | -------------------------------------------------------------------------------- /target/command.go: -------------------------------------------------------------------------------- 1 | package target 2 | 3 | import "io" 4 | 5 | type ExecCommand interface { 6 | StdoutPipe() (io.Reader, error) 7 | StderrPipe() (io.Reader, error) 8 | StdinPipe() (io.WriteCloser, error) 9 | SetStdout(io.Writer) 10 | SetStderr(io.Writer) 11 | SetStdin(io.Reader) 12 | Run() error 13 | Start() error 14 | Wait() error 15 | } 16 | -------------------------------------------------------------------------------- /target/local.go: -------------------------------------------------------------------------------- 1 | package target 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "log" 8 | "os" 9 | "os/exec" 10 | "strings" 11 | ) 12 | 13 | // Create a target for local provisioning. 14 | func NewLocalTarget() *localTarget { 15 | return &localTarget{} 16 | } 17 | 18 | type localTarget struct { 19 | cachedUser string 20 | } 21 | 22 | func (c *localTarget) String() string { 23 | return "LOCAL" 24 | } 25 | 26 | func (c *localTarget) User() string { 27 | if c.cachedUser == "" { 28 | var err error 29 | c.cachedUser, err = whoami() 30 | if err != nil { 31 | log.Fatal(err.Error()) 32 | } 33 | } 34 | return c.cachedUser 35 | } 36 | 37 | func whoami() (string, error) { 38 | u := os.Getenv("$USER") 39 | if u != "" { 40 | return u, nil 41 | } 42 | out := &bytes.Buffer{} 43 | err := &bytes.Buffer{} 44 | cmd := exec.Command("whoami") 45 | cmd.Stdout = out 46 | cmd.Stderr = err 47 | e := cmd.Run() 48 | if e != nil { 49 | return "", fmt.Errorf("error reading login name: err=%q out=%q e=%q", err.String(), out.String(), e) 50 | } 51 | return strings.TrimSpace(out.String()), nil 52 | } 53 | 54 | func (c *localTarget) Command(cmd string) (ExecCommand, error) { 55 | return &localCommand{ 56 | command: exec.Command("bash", "-c", cmd), 57 | }, nil 58 | } 59 | 60 | func (c *localTarget) Reset() (e error) { 61 | return nil 62 | } 63 | 64 | type localCommand struct { 65 | command *exec.Cmd 66 | } 67 | 68 | func (c *localCommand) StdoutPipe() (io.Reader, error) { 69 | return c.command.StdoutPipe() 70 | } 71 | 72 | func (c *localCommand) StderrPipe() (io.Reader, error) { 73 | return c.command.StderrPipe() 74 | } 75 | 76 | func (c *localCommand) StdinPipe() (io.WriteCloser, error) { 77 | return c.command.StdinPipe() 78 | } 79 | 80 | func (c *localCommand) SetStdout(w io.Writer) { 81 | c.command.Stdout = w 82 | } 83 | 84 | func (c *localCommand) SetStderr(w io.Writer) { 85 | c.command.Stderr = w 86 | } 87 | 88 | func (c *localCommand) SetStdin(r io.Reader) { 89 | c.command.Stdin = r 90 | } 91 | 92 | func (c *localCommand) Wait() error { 93 | return c.command.Wait() 94 | } 95 | 96 | func (c *localCommand) Start() error { 97 | return c.command.Start() 98 | } 99 | 100 | func (c *localCommand) Run() error { 101 | return c.command.Run() 102 | } 103 | -------------------------------------------------------------------------------- /target/ssh.go: -------------------------------------------------------------------------------- 1 | package target 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net" 7 | "os" 8 | "strconv" 9 | "strings" 10 | 11 | "golang.org/x/crypto/ssh" 12 | "golang.org/x/crypto/ssh/agent" 13 | ) 14 | 15 | func NewSshTargetWithPrivateKey(addr string, key []byte) (target *sshTarget, err error) { 16 | t, err := NewSshTarget(addr) 17 | if err != nil { 18 | return nil, err 19 | } 20 | t.key = key 21 | return t, nil 22 | } 23 | 24 | // Create a target for provisioning via SSH. 25 | func NewSshTarget(addr string) (target *sshTarget, e error) { 26 | target = &sshTarget{port: 22, user: "root"} 27 | 28 | hostAndPort := strings.SplitN(addr, ":", 2) 29 | if len(hostAndPort) == 2 { 30 | addr = hostAndPort[0] 31 | target.port, e = strconv.Atoi(hostAndPort[1]) 32 | if e != nil { 33 | return nil, fmt.Errorf("port must be given as integer, got %q", hostAndPort[1]) 34 | } 35 | } 36 | 37 | userAndAddress := strings.Split(addr, "@") 38 | switch len(userAndAddress) { 39 | case 1: 40 | target.address = addr 41 | case 2: 42 | target.user = userAndAddress[0] 43 | target.address = userAndAddress[1] 44 | default: 45 | return nil, fmt.Errorf("expected target address of the form '@', but was given: %s", addr) 46 | } 47 | 48 | if target.address == "" { 49 | e = fmt.Errorf("empty address given for target") 50 | } 51 | 52 | return target, e 53 | } 54 | 55 | type sshTarget struct { 56 | Password string 57 | 58 | user string 59 | port int 60 | address string 61 | 62 | key []byte 63 | 64 | client *ssh.Client 65 | } 66 | 67 | func (target *sshTarget) User() string { 68 | return target.user 69 | } 70 | 71 | func (target *sshTarget) String() string { 72 | return target.address 73 | } 74 | 75 | func (target *sshTarget) Command(cmd string) (ExecCommand, error) { 76 | if target.client == nil { 77 | var e error 78 | target.client, e = target.buildClient() 79 | if e != nil { 80 | return nil, e 81 | } 82 | } 83 | ses, e := target.client.NewSession() 84 | if e != nil { 85 | return nil, e 86 | } 87 | return &sshCommand{command: cmd, session: ses}, nil 88 | } 89 | 90 | func (target *sshTarget) Reset() (e error) { 91 | if target.client != nil { 92 | e = target.client.Close() 93 | target.client = nil 94 | } 95 | return e 96 | } 97 | 98 | func (target *sshTarget) buildClient() (*ssh.Client, error) { 99 | var e error 100 | config := &ssh.ClientConfig{ 101 | User: target.user, 102 | HostKeyCallback: ssh.InsecureIgnoreHostKey(), 103 | } 104 | 105 | signers := []ssh.Signer{} 106 | if target.Password != "" { 107 | config.Auth = append(config.Auth, ssh.Password(target.Password)) 108 | } else if sshSocket := os.Getenv("SSH_AUTH_SOCK"); sshSocket != "" { 109 | if agentConn, e := net.Dial("unix", sshSocket); e == nil { 110 | s, err := agent.NewClient(agentConn).Signers() 111 | if err != nil { 112 | return nil, err 113 | } 114 | signers = append(signers, s...) 115 | } 116 | } 117 | if len(target.key) > 0 { 118 | key, err := ssh.ParsePrivateKey(target.key) 119 | if err != nil { 120 | return nil, err 121 | } 122 | signers = append(signers, key) 123 | } 124 | 125 | if len(signers) > 0 { 126 | config.Auth = append(config.Auth, ssh.PublicKeys(signers...)) 127 | } 128 | 129 | con, e := ssh.Dial("tcp", fmt.Sprintf("%s:%d", target.address, target.port), config) 130 | if e != nil { 131 | return nil, e 132 | } 133 | return &ssh.Client{Conn: con}, nil 134 | } 135 | 136 | type sshCommand struct { 137 | command string 138 | session *ssh.Session 139 | } 140 | 141 | func (c *sshCommand) Close() error { 142 | return c.session.Close() 143 | } 144 | 145 | func (c *sshCommand) StdinPipe() (io.WriteCloser, error) { 146 | return c.session.StdinPipe() 147 | } 148 | 149 | func (c *sshCommand) StdoutPipe() (io.Reader, error) { 150 | return c.session.StdoutPipe() 151 | } 152 | 153 | func (c *sshCommand) StderrPipe() (io.Reader, error) { 154 | return c.session.StderrPipe() 155 | } 156 | 157 | func (c *sshCommand) SetStdout(w io.Writer) { 158 | c.session.Stdout = w 159 | } 160 | 161 | func (c *sshCommand) SetStderr(w io.Writer) { 162 | c.session.Stderr = w 163 | } 164 | 165 | func (c *sshCommand) SetStdin(r io.Reader) { 166 | c.session.Stdin = r 167 | } 168 | 169 | func (c *sshCommand) Run() error { 170 | return c.session.Run(c.command) 171 | } 172 | 173 | func (c *sshCommand) Wait() error { 174 | return c.session.Wait() 175 | } 176 | 177 | func (c *sshCommand) Start() error { 178 | return c.session.Start(c.command) 179 | } 180 | -------------------------------------------------------------------------------- /target/ssh_test.go: -------------------------------------------------------------------------------- 1 | package target 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func shouldEqualError(t *testing.T, a, b interface{}) { 8 | t.Errorf("expected %q to equal %q", a, b) 9 | } 10 | 11 | func TestNewTarget(t *testing.T) { 12 | data := map[string]struct { 13 | user, address string 14 | port int 15 | }{ 16 | "example.com": {"root", "example.com", 22}, 17 | "example.com:22": {"root", "example.com", 22}, 18 | "root@example.com": {"root", "example.com", 22}, 19 | "root@example.com:22": {"root", "example.com", 22}, 20 | "foo@example.com": {"foo", "example.com", 22}, 21 | "foo@example.com:22": {"foo", "example.com", 22}, 22 | "foo@example.com:29": {"foo", "example.com", 29}, 23 | "foo@wunderscale.com:2222": {"foo", "wunderscale.com", 2222}, 24 | } 25 | 26 | for address, expectation := range data { 27 | target, e := NewSshTarget(address) 28 | if e != nil { 29 | t.Fatalf("failed to parse address: %s", e) 30 | } 31 | 32 | if target.user != expectation.user { 33 | shouldEqualError(t, target.user, expectation.user) 34 | } 35 | 36 | if target.port != expectation.port { 37 | shouldEqualError(t, target.port, expectation.port) 38 | } 39 | 40 | if target.address != expectation.address { 41 | shouldEqualError(t, target.address, expectation.address) 42 | } 43 | } 44 | } 45 | 46 | func TestNewTargetFailing(t *testing.T) { 47 | data := map[string]string{ 48 | "": "empty address given for target", 49 | ":22": "empty address given for target", 50 | "root@:22": "empty address given for target", 51 | "example.com:foobar": `port must be given as integer, got "foobar"`, 52 | "example.com:22:23": `port must be given as integer, got "22:23"`, 53 | "root@foobar@example.com": "expected target address of the form '@', but was given: root@foobar@example.com", 54 | } 55 | 56 | for address, expectedError := range data { 57 | _, e := NewSshTarget(address) 58 | if e == nil { 59 | t.Fatalf("address %q should've invoked error %q, but didn't", address, expectedError) 60 | } 61 | 62 | if e.Error() != expectedError { 63 | shouldEqualError(t, e.Error(), expectedError) 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /task.go: -------------------------------------------------------------------------------- 1 | package urknall 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "runtime/debug" 7 | "time" 8 | 9 | "github.com/dynport/urknall/cmd" 10 | "github.com/dynport/urknall/pubsub" 11 | ) 12 | 13 | // A task is a list of commands. Each task is cached internally, i.e. if an 14 | // command has been executed already, none of the preceding tasks has changed 15 | // and neither the command itself, then it won't be executed again. This 16 | // enhances performance and removes the burden of writing idempotent commands. 17 | type Task interface { 18 | Add(cmds ...interface{}) Task 19 | Commands() ([]cmd.Command, error) 20 | } 21 | 22 | // Create a task. This is available to provide maximum flexibility, but 23 | // shouldn't be required very often. The resulting task can be registered to an 24 | // package using the AddTask method. 25 | func NewTask() Task { 26 | return &task{} 27 | } 28 | 29 | type task struct { 30 | commands []*commandWrapper 31 | 32 | name string // Name of the compilable. 33 | taskBuilder Template // only used for rendering templates TODO(gf): rename 34 | 35 | compiled bool 36 | validated bool 37 | 38 | started time.Time // time used to for caching timestamp 39 | } 40 | 41 | func (t *task) Commands() (cmds []cmd.Command, e error) { 42 | if e = t.Compile(); e != nil { 43 | return nil, e 44 | } 45 | 46 | for _, c := range t.commands { 47 | cmds = append(cmds, c.command) 48 | } 49 | 50 | return cmds, nil 51 | } 52 | 53 | func (task *task) Add(cmds ...interface{}) Task { 54 | for _, c := range cmds { 55 | switch t := c.(type) { 56 | case string: 57 | // No explicit expansion required as the function is called recursively with a ShellCommand type, that has 58 | // explicitly renders the template. 59 | task.addCommand(&stringCommand{cmd: t}) 60 | case cmd.Command: 61 | task.addCommand(t) 62 | default: 63 | panic(fmt.Sprintf("type %T not supported!", t)) 64 | } 65 | } 66 | return task 67 | } 68 | 69 | func (task *task) validate() error { 70 | if !task.validated { 71 | if task.taskBuilder == nil { 72 | return nil 73 | } 74 | e := validateTemplate(task.taskBuilder) 75 | if e != nil { 76 | return e 77 | } 78 | task.validated = true 79 | } 80 | return nil 81 | } 82 | 83 | func (task *task) addCommand(c cmd.Command) { 84 | if task.taskBuilder != nil { 85 | e := task.validate() 86 | if e != nil { 87 | panic(e.Error()) 88 | } 89 | if renderer, ok := c.(cmd.Renderer); ok { 90 | renderer.Render(task.taskBuilder) 91 | } 92 | if validator, ok := c.(cmd.Validator); ok { 93 | if e := validator.Validate(); e != nil { 94 | panic(e.Error()) 95 | } 96 | } 97 | } 98 | task.commands = append(task.commands, &commandWrapper{command: c}) 99 | } 100 | 101 | func (task *task) Compile() (e error) { 102 | if task.compiled { 103 | return nil 104 | } 105 | m := message(pubsub.MessageTasksPrecompile, "", task.name) 106 | m.Publish("started") 107 | defer func() { 108 | if r := recover(); r != nil { 109 | var ok bool 110 | e, ok = r.(error) 111 | if !ok { 112 | e = fmt.Errorf("failed to precompile package: %v %q", task.name, r) 113 | } 114 | m.Error = e 115 | m.Stack = string(debug.Stack()) 116 | m.Publish("panic") 117 | log.Printf("ERROR: %s", r) 118 | log.Print(string(debug.Stack())) 119 | } 120 | }() 121 | 122 | e = task.validate() 123 | if e != nil { 124 | return e 125 | } 126 | m.Publish("finished") 127 | task.compiled = true 128 | return nil 129 | } 130 | -------------------------------------------------------------------------------- /task_test.go: -------------------------------------------------------------------------------- 1 | package urknall 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | type vers struct { 8 | Version string 9 | } 10 | 11 | func (v *vers) Render(Package) { 12 | } 13 | 14 | func TestTaskImpl(t *testing.T) { 15 | reference := &vers{"1.2"} 16 | i := &task{taskBuilder: reference, name: "base"} 17 | i.Add("echo 1", "echo {{ .Version }}") 18 | 19 | if cmds, err := i.Commands(); err != nil { 20 | t.Errorf("didn't expect an error, got %q", err) 21 | } else if len(cmds) != 2 { 22 | t.Errorf("expected %d commands, got %q", 2, len(cmds)) 23 | } 24 | 25 | if err := i.Compile(); err != nil { 26 | t.Errorf("didn't expect an error, got %q", err) 27 | } 28 | 29 | if cmds, err := i.Commands(); err != nil { 30 | t.Errorf("didn't expect an error, got %q", err) 31 | } else if len(cmds) != 2 { 32 | t.Errorf("expected %d commands, got %q", 2, len(cmds)) 33 | } else if cmds[0].Shell() != "echo 1" { 34 | t.Errorf("expected command %d to be %q, got %q", 0, "echo 1", cmds[0]) 35 | } else if cmds[1].Shell() != "echo 1.2" { 36 | t.Errorf("expected command %d to be %q, got %q", 1, "echo 1.2", cmds[1]) 37 | } 38 | } 39 | 40 | func TestInvalidTaskImpl(t *testing.T) { 41 | reference := &struct { 42 | genericPkg 43 | Version string `urknall:"default=1.3"` 44 | }{} 45 | i := &task{taskBuilder: reference} 46 | i.Add("echo 1", "echo {{ .Version }}") 47 | 48 | if cmds, err := i.Commands(); err != nil { 49 | t.Errorf("didn't expect an error, got %q", err) 50 | } else if len(cmds) != 2 { 51 | t.Errorf("expected %d commands, got %q", 2, len(cmds)) 52 | } 53 | 54 | if err := i.Compile(); err != nil { 55 | t.Errorf("didn't expect an error, got %q", err) 56 | } 57 | 58 | if cmds, err := i.Commands(); err != nil { 59 | t.Errorf("didn't expect an error, got %q", err) 60 | } else if len(cmds) != 2 { 61 | t.Errorf("expected %d commands, got %q", 2, len(cmds)) 62 | } else if cmds[0].Shell() != "echo 1" { 63 | t.Errorf("expected command %d to be %q, got %q", 0, "echo 1", cmds[0]) 64 | } else if cmds[1].Shell() != "echo 1.3" { 65 | t.Errorf("expected command %d to be %q, got %q", 1, "echo 1.3", cmds[1]) 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /template_func.go: -------------------------------------------------------------------------------- 1 | package urknall 2 | 3 | // This is a short-cut usable for templates without configuration, i.e. where 4 | // no separate struct is required. 5 | type TemplateFunc func(Package) 6 | 7 | func (f TemplateFunc) Render(p Package) { 8 | f(p) 9 | } 10 | -------------------------------------------------------------------------------- /urknall.go: -------------------------------------------------------------------------------- 1 | // Package urknall 2 | // 3 | // See http://urknall.dynport.de for detailed documentation. 4 | package urknall 5 | 6 | import ( 7 | "io" 8 | 9 | "github.com/dynport/urknall/pubsub" 10 | ) 11 | 12 | // OpenLogger creates a logging facility for urknall using the given writer for 13 | // output. Note that the resource must be closed! 14 | func OpenLogger(w io.Writer) io.Closer { 15 | return pubsub.OpenLogger(w) 16 | } 17 | -------------------------------------------------------------------------------- /urknall/.gitattributes: -------------------------------------------------------------------------------- 1 | assets.go -diff 2 | -------------------------------------------------------------------------------- /urknall/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: build 2 | 3 | build: 4 | go get ./... 5 | 6 | -------------------------------------------------------------------------------- /urknall/base.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "sort" 7 | "strings" 8 | ) 9 | 10 | type templates map[string]*content 11 | 12 | func (t templates) exists(name string) bool { 13 | _, exist := t[name] 14 | return exist 15 | } 16 | 17 | func (t templates) names() []string { 18 | names := []string{} 19 | for n := range t { 20 | names = append(names, n) 21 | } 22 | sort.Strings(names) 23 | return names 24 | } 25 | 26 | func upstreamFiles(repo, path string) ([]*content, error) { 27 | cl := githubClient() 28 | url := "https://api.github.com/repos/" + repo + "/contents/" + path 29 | rsp, err := cl.Get(url) 30 | if err != nil { 31 | return nil, err 32 | } 33 | defer rsp.Body.Close() 34 | if rsp.Status[0] != '2' { 35 | return nil, fmt.Errorf("loading url=%s. expected status 2xx, got %q", url, rsp.Status) 36 | } 37 | decoder := json.NewDecoder(rsp.Body) 38 | 39 | contents := []*content{} 40 | return contents, decoder.Decode(&contents) 41 | } 42 | 43 | func allUpstreamTemplates(repo, path string) (tmpls templates, e error) { 44 | tmpls = templates{} 45 | contents, e := upstreamFiles(repo, path) 46 | if e != nil { 47 | return nil, e 48 | } 49 | 50 | for _, c := range contents { 51 | if strings.HasPrefix(c.Name, "tpl_") && strings.HasSuffix(c.Name, ".go") { 52 | name := c.Name[4 : len(c.Name)-3] 53 | tmpls[name] = c 54 | } 55 | } 56 | return tmpls, nil 57 | } 58 | -------------------------------------------------------------------------------- /urknall/github.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/base64" 5 | "encoding/json" 6 | "fmt" 7 | "io/ioutil" 8 | "net/http" 9 | "os" 10 | 11 | "github.com/dynport/dgtk/github" 12 | ) 13 | 14 | var b64 = base64.StdEncoding 15 | 16 | func githubClient() *http.Client { 17 | if token := os.Getenv("GITHUB_TOKEN"); token != "" { 18 | return github.NewHttpClient(token) 19 | } 20 | return &http.Client{} 21 | } 22 | 23 | func loadContent(cl *http.Client, url string, i interface{}) error { 24 | rsp, e := cl.Get(url) 25 | if e != nil { 26 | return e 27 | } 28 | defer rsp.Body.Close() 29 | b, e := ioutil.ReadAll(rsp.Body) 30 | if e != nil { 31 | return e 32 | } 33 | if rsp.Status[0] != '2' { 34 | return fmt.Errorf("expected status 2xx, got %s: %s", rsp.Status, string(b)) 35 | } 36 | return json.Unmarshal(b, &i) 37 | } 38 | 39 | type content struct { 40 | Name string `json:"name"` // "Makefile", 41 | Path string `json:"path"` // "github/Makefile", 42 | Sha string `json:"sha"` // "1a91023ae6c2b830090d615304098ac957453ae2", 43 | Size int `json:"size"` // 21, 44 | Url string `json:"url"` // "https://api.github.com/repos/dynport/dgtk/contents/github/Makefile?ref=master", 45 | HtmlUrl string `json:"html_url"` // "https://github.com/dynport/dgtk/blob/master/github/Makefile", 46 | GitUrl string `json:"git_url"` // "https://api.github.com/repos/dynport/dgtk/git/blobs/1a91023ae6c2b830090d615304098ac957453ae2", 47 | Type string `json:"type"` // "file", 48 | Content string `json:"content"` 49 | Encoding string `json:"encoding"` // "base64", 50 | } 51 | 52 | func (c *content) DecodedContent() ([]byte, error) { 53 | return b64.DecodeString(c.Content) 54 | } 55 | 56 | func (c *content) Load() error { 57 | return loadContent(githubClient(), c.Url, c) 58 | } 59 | 60 | func writeFile(p string, content []byte) error { 61 | f, e := os.OpenFile(p, os.O_CREATE|os.O_RDWR|os.O_EXCL, 0644) 62 | if e != nil { 63 | return e 64 | } 65 | defer f.Close() 66 | _, e = f.Write(content) 67 | return e 68 | } 69 | -------------------------------------------------------------------------------- /urknall/init.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "path/filepath" 8 | "strings" 9 | ) 10 | 11 | type initProject struct { 12 | Repo string `cli:"opt -r --repo default=dynport/urknall desc='repository used to retrieve files from'"` 13 | RepoPath string `cli:"opt -p --path default=examples desc='path in repository used to retrieve files from'"` 14 | 15 | BaseDir string `cli:"arg required"` 16 | } 17 | 18 | func (init *initProject) Run() error { 19 | dir, e := filepath.Abs(init.BaseDir) 20 | if e != nil { 21 | return e 22 | } 23 | 24 | _, e = os.Stat(dir) 25 | switch { 26 | case os.IsNotExist(e): 27 | if e = os.Mkdir(dir, 0755); e != nil { 28 | return e 29 | } 30 | case e != nil: 31 | return e 32 | } 33 | 34 | contents, err := upstreamFiles(init.Repo, init.RepoPath) 35 | if err != nil { 36 | return fmt.Errorf("loading upstream files: %s", err) 37 | } 38 | 39 | for _, c := range contents { 40 | localPath := dir + "/" + c.Name 41 | if strings.HasPrefix(c.Name, "cmd_") || c.Name == "main.go" { 42 | _, e := os.Stat(localPath) 43 | switch { 44 | case e == nil: 45 | logger.Printf("file %q exists", c.Name) 46 | continue 47 | case os.IsNotExist(e): 48 | e = c.Load() 49 | if e != nil { 50 | return e 51 | } 52 | 53 | content, e := c.DecodedContent() 54 | if e != nil { 55 | return e 56 | } 57 | e = writeFile(localPath, content) 58 | if e != nil { 59 | return e 60 | } 61 | 62 | logger.Printf("created %q", c.Name) 63 | default: 64 | return e 65 | } 66 | } 67 | } 68 | return nil 69 | } 70 | -------------------------------------------------------------------------------- /urknall/main.go: -------------------------------------------------------------------------------- 1 | // The urknall binary: see http://urknall.dynport.de/docs/binary/ for further information. 2 | package main 3 | 4 | import ( 5 | "log" 6 | "os" 7 | 8 | "github.com/dynport/dgtk/cli" 9 | ) 10 | 11 | var ( 12 | logger = log.New(os.Stderr, "", 0) 13 | ) 14 | 15 | func main() { 16 | e := router().RunWithArgs() 17 | switch e { 18 | case nil, cli.ErrorHelpRequested, cli.ErrorNoRoute: 19 | // ignore 20 | default: 21 | logger.Fatal(e) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /urknall/router.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "github.com/dynport/dgtk/cli" 4 | 5 | func router() *cli.Router { 6 | router := cli.NewRouter() 7 | router.Register("init", &initProject{}, "Initialize a basic urknall project.") 8 | router.Register("templates/add", &templatesAdd{}, "Add templates to project.") 9 | router.Register("templates/list", &templatesList{}, "List all available templates.") 10 | return router 11 | } 12 | -------------------------------------------------------------------------------- /urknall/templates_add.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "path/filepath" 7 | ) 8 | 9 | type templatesAdd struct { 10 | Repo string `cli:"opt -r --repo default=dynport/urknall desc='repository used to retrieve files from'"` 11 | RepoPath string `cli:"opt -p --path default=examples desc='path in repository used to retrieve files from'"` 12 | BaseDir string `cli:"opt --base-dir"` 13 | Names []string `cli:"arg required"` 14 | } 15 | 16 | func (a *templatesAdd) Run() error { 17 | tmpls, e := allUpstreamTemplates(a.Repo, a.RepoPath) 18 | if e != nil { 19 | return e 20 | } 21 | notExisting := []string{} 22 | for _, name := range a.Names { 23 | if !tmpls.exists(name) { 24 | notExisting = append(notExisting, name) 25 | } 26 | } 27 | if len(notExisting) > 0 { 28 | return fmt.Errorf("template %q does not exist. Existing names %q", notExisting, tmpls.names()) 29 | } 30 | 31 | for _, name := range a.Names { 32 | e = tmpls[name].Load() 33 | if e != nil { 34 | return e 35 | } 36 | content, e := tmpls[name].DecodedContent() 37 | if e != nil { 38 | return e 39 | } 40 | 41 | if e = ioutil.WriteFile(filepath.Join(a.BaseDir, "tpl_"+name+".go"), content, 0644); e != nil { 42 | return e 43 | } 44 | } 45 | return nil 46 | } 47 | -------------------------------------------------------------------------------- /urknall/templates_list.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "fmt" 4 | 5 | type templatesList struct { 6 | Repo string `cli:"opt -r --repo default=dynport/urknall desc='repository used to retrieve files from'"` 7 | RepoPath string `cli:"opt -p --path default=examples desc='path in repository used to retrieve files from'"` 8 | } 9 | 10 | func (list *templatesList) Run() error { 11 | all, e := allUpstreamTemplates(list.Repo, list.RepoPath) 12 | if e != nil { 13 | return e 14 | } 15 | fmt.Println("available templates: ") 16 | for _, name := range all.names() { 17 | fmt.Println(" * " + name) 18 | } 19 | return nil 20 | } 21 | -------------------------------------------------------------------------------- /utils.go: -------------------------------------------------------------------------------- 1 | package urknall 2 | 3 | import ( 4 | "crypto/sha256" 5 | "fmt" 6 | 7 | "github.com/dynport/urknall/cmd" 8 | ) 9 | 10 | func renderTemplate(builder Template) (*packageImpl, error) { 11 | p := &packageImpl{reference: builder} 12 | e := validateTemplate(builder) 13 | if e != nil { 14 | return nil, e 15 | } 16 | builder.Render(p) 17 | return p, nil 18 | } 19 | 20 | func commandChecksum(c cmd.Command) (string, error) { 21 | if c, ok := c.(interface { 22 | Checksum() string 23 | }); ok { 24 | return c.Checksum(), nil 25 | } 26 | s := sha256.New() 27 | if _, e := s.Write([]byte(c.Shell())); e != nil { 28 | return "", e 29 | } 30 | return fmt.Sprintf("%x", s.Sum(nil)), nil 31 | } 32 | 33 | func midTrunc(in string, l int) string { 34 | if len(in) <= l { 35 | return in 36 | } 37 | chars := l - 3 38 | beg := chars / 2 39 | end := beg 40 | if chars%2 == 1 { 41 | beg++ 42 | } 43 | return in[0:beg] + "..." + in[len(in)-end:] 44 | } 45 | -------------------------------------------------------------------------------- /utils/templates.go: -------------------------------------------------------------------------------- 1 | // Util functions that don't fit anywhere else. 2 | package utils 3 | 4 | import ( 5 | "bytes" 6 | "fmt" 7 | "text/template" 8 | ) 9 | 10 | // Delegates action to RenderTemplate. Panics in case of an error. 11 | func MustRenderTemplate(tmplString string, i interface{}) (rendered string) { 12 | for j := 0; j < 8; j++ { 13 | renderedCommand, e := RenderTemplate(tmplString, i) 14 | if e != nil { 15 | panic(fmt.Errorf("failed rendering template: %s (%s)", e.Error(), tmplString)) 16 | } 17 | if renderedCommand == tmplString { 18 | return renderedCommand 19 | } 20 | tmplString = renderedCommand 21 | } 22 | panic("found rendering loop. max 8 levels are allowed") 23 | } 24 | 25 | // Render the template from the given string using text/template and the 26 | // information from the interface provided. 27 | func RenderTemplate(tmplString string, i interface{}) (rendered string, e error) { 28 | tpl := template.New("") 29 | tpl, e = tpl.Parse(tmplString) 30 | if e != nil { 31 | return "", e 32 | } 33 | 34 | resultBuffer := &bytes.Buffer{} 35 | e = tpl.Execute(resultBuffer, i) 36 | return string(resultBuffer.Bytes()), e 37 | } 38 | -------------------------------------------------------------------------------- /utils/templates_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestMustRender(t *testing.T) { 8 | type typ struct { 9 | Version string 10 | Path string 11 | } 12 | ins := typ{Version: "1.2.3", Path: "/path/to/{{ .Version }}"} 13 | res := MustRenderTemplate("{{ .Path }}", ins) 14 | if res != "/path/to/1.2.3" { 15 | t.Errorf("expected result to be %q, got %q", "/path/to/1.2.3", res) 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /utils/version_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestParseVersion(t *testing.T) { 8 | v := &Version{} 9 | if err := v.Parse("1.2.3"); err != nil { 10 | t.Errorf("didn't expect an error, got %q", err) 11 | } 12 | 13 | if v.Major != 1 { 14 | t.Errorf("expected major version to be %d, got %d", 1, v.Major) 15 | } 16 | if v.Minor != 2 { 17 | t.Errorf("expected major version to be %d, got %d", 2, v.Minor) 18 | } 19 | if v.Patch != 3 { 20 | t.Errorf("expected major version to be %d, got %d", 3, v.Patch) 21 | } 22 | } 23 | 24 | func TestCompareSmallerVersion(t *testing.T) { 25 | a, err := ParseVersion("0.1.2") 26 | if err != nil { 27 | t.Errorf("didn't expect an error, got %q", err) 28 | } 29 | 30 | for _, v := range []string{"0.1.3", "0.2.0", "1.0.0"} { 31 | b, err := ParseVersion(v) 32 | if err != nil { 33 | t.Errorf("didn't expect an error, got %q", err) 34 | } 35 | if !a.Smaller(b) { 36 | t.Errorf("expected %q to be smaller than %q, wasn't!", a, b) 37 | } 38 | } 39 | } 40 | 41 | func TestCompareEqualVersions(t *testing.T) { 42 | a, err := ParseVersion("0.1.2") 43 | if err != nil { 44 | t.Errorf("didn't expect an error, got %q", err) 45 | } 46 | 47 | b, err := ParseVersion("0.1.2") 48 | if err != nil { 49 | t.Errorf("didn't expect an error, got %q", err) 50 | } 51 | if a.Smaller(b) { 52 | t.Errorf("expected %q to be smaller than %q, wasn't!", a, b) 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /utils/versions.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "strings" 7 | ) 8 | 9 | type Version struct { 10 | Major int 11 | Minor int 12 | Patch int 13 | } 14 | 15 | func (version *Version) Parse(raw string) error { 16 | parts := strings.Split(raw, ".") 17 | if len(parts) == 3 { 18 | version.Major, _ = strconv.Atoi(parts[0]) 19 | version.Minor, _ = strconv.Atoi(parts[1]) 20 | version.Patch, _ = strconv.Atoi(parts[2]) 21 | return nil 22 | } 23 | return fmt.Errorf("could not parse %s into version", raw) 24 | } 25 | 26 | func (version *Version) String() string { 27 | return fmt.Sprintf("%d.%d.%d", version.Major, version.Minor, version.Patch) 28 | } 29 | 30 | func ParseVersion(raw string) (v *Version, e error) { 31 | v = &Version{} 32 | e = v.Parse(raw) 33 | return v, e 34 | } 35 | 36 | func (version *Version) Smaller(other *Version) bool { 37 | if version.Major < other.Major { 38 | return true 39 | } 40 | if version.Minor < other.Minor { 41 | return true 42 | } 43 | if version.Patch < other.Patch { 44 | return true 45 | } 46 | return false 47 | } 48 | -------------------------------------------------------------------------------- /utils_test.go: -------------------------------------------------------------------------------- 1 | package urknall 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | type testCommandCustomChecksum struct { 8 | *testCommand 9 | } 10 | 11 | func (c *testCommandCustomChecksum) Checksum() string { 12 | return "default checksum" 13 | } 14 | 15 | func TestUtils(t *testing.T) { 16 | c1 := &testCommand{} 17 | if checksum, err := commandChecksum(c1); err != nil { 18 | t.Errorf("didn't expect an error, got %q", err) 19 | } else if checksum != "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" { 20 | t.Errorf("expected %q, got %q", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", checksum) 21 | } 22 | 23 | c2 := &testCommandCustomChecksum{c1} 24 | if checksum, err := commandChecksum(c2); err != nil { 25 | t.Errorf("didn't expect an error, got %q", err) 26 | } else if checksum != "default checksum" { 27 | t.Errorf("expected %q, got %q", "default checksum", checksum) 28 | } 29 | } 30 | 31 | func TestMidTruncate(t *testing.T) { 32 | tests := []struct { 33 | In string 34 | Len int 35 | Out string 36 | }{ 37 | {"test", 4, "test"}, 38 | {"abcdef", 5, "a...f"}, 39 | {"abcdefg", 6, "ab...g"}, 40 | } 41 | 42 | for _, tst := range tests { 43 | if v, ex := midTrunc(tst.In, tst.Len), tst.Out; ex != v { 44 | t.Errorf("expected midTrunc of %q with len %d to be %q, was %q", tst.In, tst.Len, ex, v) 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /validation.go: -------------------------------------------------------------------------------- 1 | package urknall 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "strconv" 7 | "strings" 8 | ) 9 | 10 | const ( 11 | parse_INT_ERROR = `failed to parse value (not an int) of tag %q: "%s"` 12 | parse_BOOL_ERROR = `failed to parse value (neither "true" nor "false") of tag %q: "%s"` 13 | unknown_TAG_ERROR = `type %q doesn't support %q tag` 14 | ) 15 | 16 | type validationOptions struct { 17 | required bool 18 | defaultValue interface{} 19 | size int64 20 | min int64 21 | max int64 22 | } 23 | 24 | func validateTemplate(pkg Template) error { 25 | v := reflect.ValueOf(pkg) 26 | if v.Kind() == reflect.Ptr { 27 | v = v.Elem() 28 | } 29 | 30 | if v.Kind() != reflect.Struct { 31 | return nil 32 | } 33 | 34 | for i := 0; i < v.NumField(); i++ { 35 | e := validateField(v.Type().Field(i), v.Field(i)) 36 | if e != nil { 37 | return fmt.Errorf("[package:%s]%s", v.Type().Name(), e.Error()) 38 | } 39 | } 40 | return nil 41 | } 42 | 43 | func validateField(field reflect.StructField, value reflect.Value) error { 44 | opts, e := parseFieldValidationString(field) 45 | if e != nil { 46 | return fmt.Errorf("[field:%s] %s", field.Name, e.Error()) 47 | } 48 | 49 | switch field.Type.String() { 50 | case "[]uint8": 51 | if opts.required && len(value.Bytes()) == 0 { 52 | return fmt.Errorf("[field:%s] required field not set", field.Name) 53 | } 54 | if opts.defaultValue != nil && len(value.Bytes()) == 0 { 55 | value.SetBytes([]byte(opts.defaultValue.(string))) 56 | } 57 | return nil 58 | case "[]string": 59 | if opts.required && value.Len() == 0 { 60 | return fmt.Errorf("[field:%s] required field not set", field.Name) 61 | } 62 | return nil 63 | case "string": 64 | if opts.required && value.String() == "" { 65 | return fmt.Errorf("[field:%s] required field not set", field.Name) 66 | } 67 | 68 | if opts.defaultValue != nil && value.String() == "" { 69 | value.SetString(opts.defaultValue.(string)) 70 | } 71 | return validateString(field, value.String(), opts) 72 | case "int": 73 | if opts.defaultValue != nil && value.Int() == 0 { 74 | value.SetInt(opts.defaultValue.(int64)) 75 | } 76 | return validateInt(field, value.Int(), opts) 77 | case "bool": 78 | if opts.defaultValue != nil && opts.defaultValue.(bool) { 79 | value.SetBool(true) 80 | } 81 | } 82 | return nil 83 | } 84 | 85 | func parseFieldValidationString(field reflect.StructField) (opts *validationOptions, e error) { 86 | opts = &validationOptions{} 87 | tagString := field.Tag.Get("urknall") 88 | 89 | fields := []string{} 90 | idxStart := 0 91 | sA := false 92 | for i, c := range tagString { 93 | if c == '\'' { 94 | sA = !sA 95 | } 96 | 97 | if (c == ' ' || i+1 == len(tagString)) && !sA { 98 | fields = append(fields, tagString[idxStart:i+1]) 99 | idxStart = i + 1 100 | } 101 | } 102 | if sA { 103 | return nil, fmt.Errorf("failed to parse tag due to erroneous quotes") 104 | } 105 | 106 | for fIdx := range fields { 107 | kvList := strings.SplitN(fields[fIdx], "=", 2) 108 | if len(kvList) != 2 { 109 | return nil, fmt.Errorf("failed to parse annotation (value missing): %q", fields[fIdx]) 110 | } 111 | key := strings.TrimSpace(kvList[0]) 112 | value := strings.Trim(kvList[1], " '") 113 | 114 | switch key { 115 | case "required": 116 | switch field.Type.String() { 117 | case "string", "[]string", "[]uint8": 118 | if value != "true" && value != "false" { 119 | return nil, fmt.Errorf(parse_BOOL_ERROR, key, value) 120 | } 121 | opts.required = value == "true" 122 | default: 123 | return nil, fmt.Errorf(unknown_TAG_ERROR, field.Type.String(), key) 124 | } 125 | case "default": 126 | switch field.Type.String() { 127 | case "string", "[]uint8": 128 | opts.defaultValue = value 129 | case "int": 130 | i, e := strconv.ParseInt(value, 10, 64) 131 | if e != nil { 132 | return nil, fmt.Errorf(parse_INT_ERROR, key, value) 133 | } 134 | opts.defaultValue = i 135 | case "bool": 136 | if value != "true" && value != "false" { 137 | return nil, fmt.Errorf(parse_BOOL_ERROR, key, value) 138 | } 139 | opts.defaultValue = value == "true" 140 | default: 141 | return nil, fmt.Errorf(unknown_TAG_ERROR, field.Type.String(), key) 142 | } 143 | case "size": 144 | switch field.Type.String() { 145 | case "string": 146 | i, e := strconv.ParseInt(value, 10, 64) 147 | if e != nil { 148 | return nil, fmt.Errorf(parse_INT_ERROR, key, value) 149 | } 150 | opts.size = i 151 | default: 152 | return nil, fmt.Errorf(unknown_TAG_ERROR, field.Type.String(), key) 153 | } 154 | case "min": 155 | switch field.Type.String() { 156 | case "string": 157 | fallthrough 158 | case "int": 159 | i, e := strconv.ParseInt(value, 10, 64) 160 | if e != nil { 161 | return nil, fmt.Errorf(parse_INT_ERROR, key, value) 162 | } 163 | opts.min = i 164 | default: 165 | return nil, fmt.Errorf(unknown_TAG_ERROR, field.Type.String(), key) 166 | } 167 | case "max": 168 | switch field.Type.String() { 169 | case "string": 170 | fallthrough 171 | case "int": 172 | i, e := strconv.ParseInt(value, 10, 64) 173 | if e != nil { 174 | return nil, fmt.Errorf(parse_INT_ERROR, key, value) 175 | } 176 | opts.max = i 177 | default: 178 | return nil, fmt.Errorf(unknown_TAG_ERROR, field.Type.String(), key) 179 | } 180 | default: 181 | return nil, fmt.Errorf(`tag %q unknown`, key) 182 | } 183 | } 184 | return opts, nil 185 | } 186 | 187 | func validateInt(field reflect.StructField, value int64, opts *validationOptions) (e error) { 188 | if opts.min != 0 && value < opts.min { 189 | return fmt.Errorf(`[field:%s] value "%d" smaller than the specified minimum "%d"`, field.Name, value, opts.min) 190 | } 191 | 192 | if opts.max != 0 && value > opts.max { 193 | return fmt.Errorf(`[field:%s] value "%d" greater than the specified maximum "%d"`, field.Name, value, opts.max) 194 | } 195 | 196 | return nil 197 | } 198 | 199 | func validateString(field reflect.StructField, value string, opts *validationOptions) (e error) { 200 | if opts.min != 0 && value != "" && (int64(len(value))) < opts.min { 201 | return fmt.Errorf(`[field:%s] length of value %q smaller than the specified minimum length "%d"`, field.Name, value, opts.min) 202 | } 203 | 204 | if opts.max != 0 && int64(len(value)) > opts.max { 205 | return fmt.Errorf(`[field:%s] length of value %q greater than the specified maximum length "%d"`, field.Name, value, opts.max) 206 | } 207 | 208 | if opts.size != 0 && value != "" && int64(len(value)) != opts.size { 209 | return fmt.Errorf(`[field:%s] length of value %q doesn't match the specified size "%d"`, field.Name, value, opts.size) 210 | } 211 | 212 | return nil 213 | } 214 | --------------------------------------------------------------------------------