├── .github ├── building.gif └── tty.gif ├── .gitignore ├── .gitmodules ├── LICENSE ├── Makefile ├── Makefile.gobuild ├── README.md ├── data ├── 00_solbuild.conf ├── 99_unstable.conf ├── local-unstable-x86_64.profile ├── main-x86_64.profile └── unstable-x86_64.profile ├── man ├── solbuild.1 ├── solbuild.1.html ├── solbuild.1.md ├── solbuild.conf.5 ├── solbuild.conf.5.html ├── solbuild.conf.5.md ├── solbuild.profile.5 ├── solbuild.profile.5.html └── solbuild.profile.5.md └── src ├── builder ├── build.go ├── chroot.go ├── config.go ├── copy.go ├── eopkg.go ├── history.go ├── index.go ├── lockfile.go ├── main.go ├── manager.go ├── mmap.go ├── namespaces.go ├── overlay.go ├── pkg.go ├── profile.go ├── profile_test.go ├── repos.go ├── source │ ├── git.go │ ├── main.go │ └── simple.go ├── testdata │ ├── group │ ├── passwd │ └── unstable.profile ├── transit_manifest.go ├── update.go ├── userinfo.go ├── users.go ├── users_test.go └── util.go └── solbuild ├── cmd ├── build.go ├── chroot.go ├── delete_cache.go ├── index.go ├── init.go ├── root.go ├── update.go └── version.go └── main.go /.github/building.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/solus-project/solbuild/15156176b5da5057ba2e0742c629a7b41af01e76/.github/building.gif -------------------------------------------------------------------------------- /.github/tty.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/solus-project/solbuild/15156176b5da5057ba2e0742c629a7b41af01e76/.github/tty.gif -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /bin/ 2 | /pkg/ 3 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "src/vendor/github.com/solus-project/libosdev"] 2 | path = src/vendor/github.com/solus-project/libosdev 3 | url = https://github.com/solus-project/libosdev.git 4 | [submodule "src/vendor/github.com/Sirupsen/logrus"] 5 | path = src/vendor/github.com/Sirupsen/logrus 6 | url = https://github.com/Sirupsen/logrus.git 7 | [submodule "src/vendor/github.com/spf13/pflag"] 8 | path = src/vendor/github.com/spf13/pflag 9 | url = https://github.com/spf13/pflag.git 10 | [submodule "src/vendor/github.com/spf13/cobra"] 11 | path = src/vendor/github.com/spf13/cobra 12 | url = https://github.com/spf13/cobra.git 13 | [submodule "src/vendor/github.com/go-yaml/yaml"] 14 | path = src/vendor/github.com/go-yaml/yaml 15 | url = https://github.com/go-yaml/yaml.git 16 | [submodule "src/vendor/github.com/go-ini/ini"] 17 | path = src/vendor/github.com/go-ini/ini 18 | url = https://github.com/go-ini/ini.git 19 | [submodule "src/vendor/github.com/BurntSushi/toml"] 20 | path = src/vendor/github.com/BurntSushi/toml 21 | url = https://github.com/BurntSushi/toml.git 22 | [submodule "src/vendor/github.com/libgit2/git2go"] 23 | path = src/vendor/github.com/libgit2/git2go 24 | url = https://github.com/libgit2/git2go.git 25 | [submodule "src/vendor/github.com/cheggaaa/pb"] 26 | path = src/vendor/github.com/cheggaaa/pb 27 | url = https://github.com/cheggaaa/pb.git 28 | [submodule "src/vendor/github.com/mattn/go-runewidth"] 29 | path = src/vendor/github.com/mattn/go-runewidth 30 | url = https://github.com/mattn/go-runewidth.git 31 | [submodule "src/vendor/github.com/andelf/go-curl"] 32 | path = src/vendor/github.com/andelf/go-curl 33 | url = https://github.com/andelf/go-curl.git 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PROJECT_ROOT := src/ 2 | VERSION = 1.4.2 3 | 4 | .DEFAULT_GOAL := all 5 | 6 | # The resulting binaries map to the subproject names 7 | BINARIES = \ 8 | solbuild 9 | 10 | GO_TESTS = \ 11 | builder.test 12 | 13 | include Makefile.gobuild 14 | 15 | _PKGS = \ 16 | builder \ 17 | builder/source \ 18 | solbuild \ 19 | solbuild/cmd 20 | 21 | # We want to add compliance for all built binaries 22 | _CHECK_COMPLIANCE = $(addsuffix .compliant,$(_PKGS)) 23 | 24 | # Build all binaries as static binary 25 | BINS = $(addsuffix .statbin,$(BINARIES)) 26 | 27 | # Ensure our own code is compliant.. 28 | compliant: $(_CHECK_COMPLIANCE) 29 | install: $(BINS) 30 | test -d $(DESTDIR)/usr/bin || install -D -d -m 00755 $(DESTDIR)/usr/bin; \ 31 | install -m 00755 bin/* $(DESTDIR)/usr/bin/.; \ 32 | test -d $(DESTDIR)/usr/share/solbuild || install -D -d -m 00755 $(DESTDIR)/usr/share/solbuild; \ 33 | install -m 00644 data/*.profile $(DESTDIR)/usr/share/solbuild/.; 34 | install -m 00644 data/00_solbuild.conf $(DESTDIR)/usr/share/solbuild/.; 35 | test -d $(DESTDIR)/usr/share/man/man1 || install -D -d -m 00755 $(DESTDIR)/usr/share/man/man1; \ 36 | install -m 00644 man/*.1 $(DESTDIR)/usr/share/man/man1/.; \ 37 | test -d $(DESTDIR)/usr/share/man/man5 || install -D -d -m 00755 $(DESTDIR)/usr/share/man/man5; \ 38 | install -m 00644 man/*.5 $(DESTDIR)/usr/share/man/man5/.; 39 | 40 | 41 | ensure_modules: 42 | @ ( \ 43 | git submodule init; \ 44 | git submodule update; \ 45 | ); 46 | 47 | # Credit to swupd developers: https://github.com/clearlinux/swupd-client 48 | MANPAGES = \ 49 | man/solbuild.1 \ 50 | man/solbuild.conf.5 \ 51 | man/solbuild.profile.5 52 | 53 | gen_docs: 54 | for MANPAGE in $(MANPAGES); do \ 55 | ronn --roff < $${MANPAGE}.md > $${MANPAGE}; \ 56 | ronn --html < $${MANPAGE}.md > $${MANPAGE}.html; \ 57 | done 58 | 59 | # See: https://github.com/meitar/git-archive-all.sh/blob/master/git-archive-all.sh 60 | release: ensure_modules 61 | git-archive-all.sh --format tar.gz --prefix solbuild-$(VERSION)/ --verbose -t HEAD solbuild-$(VERSION).tar.gz 62 | 63 | all: $(BINS) 64 | -------------------------------------------------------------------------------- /Makefile.gobuild: -------------------------------------------------------------------------------- 1 | CUR_DIR = $(shell pwd) 2 | 3 | # "Normal" static binary 4 | %.statbin: 5 | GOPATH=$(CUR_DIR) go install -v $(subst .statbin,,$@) 6 | 7 | clean: 8 | test ! -d $(CUR_DIR)/pkg || rm -rvf $(CUR_DIR)/pkg; \ 9 | test ! -d $(CUR_DIR)/bin || rm -rvf $(CUR_DIR)/bin 10 | 11 | spellcheck: 12 | @ ( \ 13 | misspell -error -i 'evolveos' `find $(PROJECT_ROOT) -not -path '*/vendor/*' -name '*.go'`; \ 14 | ); 15 | 16 | %.compliant: spellcheck 17 | @ ( \ 18 | cd "$(PROJECT_ROOT)/$(subst .compliant,,$@)" >/dev/null || exit 1; \ 19 | go fmt || exit 1; \ 20 | GOPATH=$(CUR_DIR)/ golint || exit 1; \ 21 | GOPATH=$(CUR_DIR)/ go vet || exit 1; \ 22 | ); 23 | 24 | %.test: 25 | GOPATH=$(CUR_DIR) go test $(subst .test,,$@) 26 | 27 | check: $(GO_TESTS) 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | solbuild 2 | -------- 3 | 4 | [![Report](https://goreportcard.com/badge/github.com/solus-project/solbuild)](https://goreportcard.com/report/github.com/solus-project/solbuild) [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) 5 | 6 | `solbuild` is a `chroot` based package build system, used to safely and efficiently build Solus packages from source, in a highly controlled and isolated environment. This tool succeeds the `evobuild` tool, originally in Evolve OS, which is now known as Solus. The very core concept of the layered builder has always remained the same, this is the next .. evolution.. of the tool. 7 | 8 | ![tty](https://raw.githubusercontent.com/solus-project/solbuild/master/.github/building.gif) 9 | 10 | `solbuild` makes use of `OverlayFS` to provide a simple caching system, whereby a base image (provided by the Solus project) is used as the bottom-most, read-only layer, and changes are made in temporary upper layers. Currently the project provides two base images for the default profiles shipped with `solbuild`: 11 | 12 | - **main-x86_64**: Built using the stable Solus repositories, suitable for production deployments for `shannon` users. 13 | - **unstable-x86_64**: Built using the unstable Solus repositories, ideal for developers, and the Solus build machinery for the repo waterfall prior to `shannon` inclusion. 14 | 15 | When building `package.yml` files ([ypkg](https://github.com/solus-project/ypkg)), the tool will also disable all networking within the environment, apart from the loopback device. This is intended to prevent uncontrolled build environments in which a package may be fetching external, unverified sources, during the build. 16 | 17 | `solbuild` also allows developers to control the repositories used by configuring the profiles: 18 | 19 | - Remove any base image repo 20 | - Add any given repo, remote or local. Local repos are bind mounted and can be automatically indexed by `solbuild`. 21 | 22 | `solbuild` performs heavy caching throughout, with source archives being stored in unique hash based directories globally, and the `ccache` being retained after each build through bind mounts. A single package cache is retained to speed up subsequent builds, and users may speed this up further by updating their base images. 23 | 24 | As a last speed booster, `solbuild` allows you to perform builds in memory via the `--tmpfs` option. 25 | 26 | **Note**: `solbuild` is designed in such a way that you *do not need to be running Solus*. You can build packages for Solus from any compatible host. 27 | 28 | 29 | solbuild is a [Solus project](https://solus-project.com/). 30 | 31 | ![logo](https://build.solus-project.com/logo.png) 32 | 33 | 34 | Getting started 35 | ---------------- 36 | 37 | **Solus Users** 38 | 39 | sudo eopkg up 40 | sudo eopkg it solbuild 41 | 42 | # If you only ever want to use the unstable repo by default 43 | sudo eopkg it solbuild-config-unstable 44 | 45 | **Everyone else** 46 | 47 | git clone https://github.com/solus-project/solbuild.git 48 | cd solbuild 49 | make ensure_modules 50 | make 51 | sudo make install 52 | 53 | You may wish to use the provided tarballs, which include vendored dependencies. 54 | Distributions are free to nuke the src/vendor directory from the distributed 55 | tarball and use their own golang dependencies if appropriate. 56 | 57 | **Initialising the root** 58 | 59 | Run the following command to fetch and install the base image. If you wish 60 | to change the profile, use the `-p` flag (`unstable-x86_64` or `main-x86_64`) 61 | The `-u` flag will automatically update the image. 62 | 63 | sudo solbuild init -u 64 | 65 | **Updating the image** 66 | 67 | # Update the default profile 68 | sudo solbuild update 69 | 70 | # Update a specific profile 71 | sudo solbuild update unstable-x86_64 72 | 73 | **Building packages** 74 | 75 | # Build the first package found in the current directory 76 | sudo solbuild build 77 | 78 | # Build a specific path 79 | sudo solbuild build ../mypackages/package.yml 80 | 81 | # Build for unstable profile 82 | sudo solbuild -p unstable-x86_64 build 83 | 84 | See the `solbuild help` command for more details, or `solbuild(1)` manpage. 85 | 86 | Requirements 87 | ------------ 88 | 89 | - golang (tested with 1.7.4) 90 | - `libgit2` (Also require `git` at runtime for submodules) 91 | - `curl` command 92 | 93 | Your kernel must support the `overlayfs` filesystem. 94 | Git is required as `solbuild` supports the `git|` source type of ypkg files. Additionally, `solbuild` will try to generate a package changelog from the git history where the YPKG file is found. This is used within Solus to create a changelog dynamically from the git tags, and automatically marking security updates, etc. 95 | 96 | License 97 | ------- 98 | 99 | Copyright © 2016-2017 Solus Project 100 | 101 | `solbuild` is available under the terms of the Apache-2.0 license 102 | -------------------------------------------------------------------------------- /data/00_solbuild.conf: -------------------------------------------------------------------------------- 1 | # 2 | # solbuild default configuration 3 | # 4 | # Do not make changes to this file. solbuild is implemented in a stateless 5 | # fashion, and will load files in a layered mechanism. If you wish to edit 6 | # this configuration, copy to /etc/solbuild/. 7 | # 8 | 9 | # The default profile will be used in the absence of a 10 | # "-p" profile argument to solbuild 11 | default_profile = "main-x86_64" 12 | 13 | # Setting this to true will default the builder to using tmpfs 14 | # Note you can still override this at runtime with the -t flag 15 | enable_tmpfs = false 16 | 17 | # This is passed directly to mount, and is the "-o size=" argument 18 | # for mounting a tmpfs. Good value would be: 2G. An empty size will 19 | # mean an unbounded tmpfs size. 20 | tmpfs_size = "" 21 | -------------------------------------------------------------------------------- /data/99_unstable.conf: -------------------------------------------------------------------------------- 1 | # 2 | # solbuild unstable configuration 3 | # 4 | # This configuration will force the default profile to be 5 | # unstable-x86_64 6 | # 7 | 8 | # The default profile will be used in the absence of a 9 | # "-p" profile argument to solbuild 10 | default_profile = "unstable-x86_64" 11 | -------------------------------------------------------------------------------- /data/local-unstable-x86_64.profile: -------------------------------------------------------------------------------- 1 | # 2 | # local-unstable-x86_64 configuration 3 | # 4 | # Build Solus packages using the unstable repository image. 5 | # This is the default profile for the Solus build server and developers. 6 | # 7 | # Do not make changes to this file. solbuild is implemented in a stateless 8 | # fashion, and will load files in a layered mechanism. If you wish to edit 9 | # this profile, copy to /etc/solbuild/. 10 | # 11 | # It is generally advisable to create a *new* profile name in /etc, because 12 | # we will load /etc/ before /usr/share. Thus, profiles with the same name 13 | # in /etc/ are loaded *first* and will override this profile. 14 | # 15 | # Of course, if that's what you intended to do, then by all means, do so. 16 | 17 | image = "unstable-x86_64" 18 | 19 | # If you have a local repo providing packages that exist in the main 20 | # repository already, you should remove the repo, and re-add it *after* 21 | # your local repository: 22 | remove_repos = ['Solus'] 23 | add_repos = ['Local','Solus'] 24 | 25 | # A local repo with automatic indexing 26 | [repo.Local] 27 | uri = "/var/lib/solbuild/local" 28 | local = true 29 | autoindex = true 30 | 31 | # Re-add the Solus unstable repo 32 | [repo.Solus] 33 | uri = "https://packages.solus-project.com/unstable/eopkg-index.xml.xz" 34 | -------------------------------------------------------------------------------- /data/main-x86_64.profile: -------------------------------------------------------------------------------- 1 | # 2 | # main-x86_64 configuration 3 | # 4 | # Build Solus packages using the stable repository image. 5 | # Use this profile if you are on the stable repository and are building packages 6 | # for yourself, or you are a vendor deploying .eopkg's for stable repo users. 7 | # 8 | # Do not make changes to this file. solbuild is implemented in a stateless 9 | # fashion, and will load files in a layered mechanism. If you wish to edit 10 | # this profile, copy to /etc/solbuild/. 11 | # 12 | # It is generally advisable to create a *new* profile name in /etc, because 13 | # we will load /etc/ before /usr/share. Thus, profiles with the same name 14 | # in /etc/ are loaded *first* and will override this profile. 15 | # 16 | # Of course, if that's what you intended to do, then by all means, do so. 17 | 18 | image = "main-x86_64" 19 | 20 | # Remove all the repos from the base image 21 | # remove_repos = ['*'] 22 | 23 | # Remove just a single repo from the base image 24 | # remove_repos = ['Solus'] 25 | 26 | # Restrict enabled repos to just one repo 27 | # add_repos = ["Solus"] 28 | 29 | # If you have a local repo providing packages that exist in the main 30 | # repository already, you should remove the repo, and re-add it *after* 31 | # your local repository: 32 | # remove_repos = ['Solus'] 33 | # add_repos = ['Local','Solus'] 34 | 35 | # Example of adding a remote repo 36 | # [repo.Solus] 37 | # uri = "https://packages.solus-project.com/unstable/eopkg-index.xml.xz" 38 | 39 | # Add a local repository by bind mounting it into chroot on each build 40 | # [repo.Local] 41 | # uri = "/var/lib/myrepo" 42 | # local = true 43 | 44 | # A local repo with automatic indexing 45 | # [repo.LocalIndexed] 46 | # uri = "/var/lib/myOtherRepo" 47 | # local = true 48 | # autoindex = true 49 | -------------------------------------------------------------------------------- /data/unstable-x86_64.profile: -------------------------------------------------------------------------------- 1 | # 2 | # unstable-x86_64 configuration 3 | # 4 | # Build Solus packages using the unstable repository image. 5 | # This is the default profile for the Solus build server and developers. 6 | # 7 | # Do not make changes to this file. solbuild is implemented in a stateless 8 | # fashion, and will load files in a layered mechanism. If you wish to edit 9 | # this profile, copy to /etc/solbuild/. 10 | # 11 | # It is generally advisable to create a *new* profile name in /etc, because 12 | # we will load /etc/ before /usr/share. Thus, profiles with the same name 13 | # in /etc/ are loaded *first* and will override this profile. 14 | # 15 | # Of course, if that's what you intended to do, then by all means, do so. 16 | 17 | image = "unstable-x86_64" 18 | 19 | # Remove all the repos from the base image 20 | # remove_repos = ['*'] 21 | 22 | # Remove just a single repo from the base image 23 | # remove_repos = ['Solus'] 24 | 25 | # Restrict enabled repos to just one repo 26 | # add_repos = ["Solus"] 27 | 28 | # If you have a local repo providing packages that exist in the main 29 | # repository already, you should remove the repo, and re-add it *after* 30 | # your local repository: 31 | # remove_repos = ['Solus'] 32 | # add_repos = ['Local','Solus'] 33 | 34 | # Example of adding a remote repo 35 | # [repo.Solus] 36 | # uri = "https://packages.solus-project.com/unstable/eopkg-index.xml.xz" 37 | 38 | # Add a local repository by bind mounting it into chroot on each build 39 | # [repo.Local] 40 | # uri = "/var/lib/myrepo" 41 | # local = true 42 | 43 | # A local repo with automatic indexing 44 | # [repo.LocalIndexed] 45 | # uri = "/var/lib/myOtherRepo" 46 | # local = true 47 | # autoindex = true 48 | 49 | -------------------------------------------------------------------------------- /man/solbuild.1: -------------------------------------------------------------------------------- 1 | .\" generated with Ronn/v0.7.3 2 | .\" http://github.com/rtomayko/ronn/tree/0.7.3 3 | . 4 | .TH "SOLBUILD" "1" "January 2017" "" "" 5 | . 6 | .SH "NAME" 7 | \fBsolbuild\fR \- Solus package builder 8 | . 9 | .SH "SYNOPSIS" 10 | \fBsolbuild [subcommand] \fR 11 | . 12 | .SH "DESCRIPTION" 13 | \fBsolbuild(1)\fR is a \fBchroot(2)\fR based package build system, used to safely and efficiently build Solus packages from source\. 14 | . 15 | .P 16 | \fBsolbuild(1)\fR makes use of \fBOverlayFS\fR to provide a simple caching system, whereby a base image (provided by the Solus project) is used as the bottom\-most, read\-only layer, and changes are made in temporary upper layers\. 17 | . 18 | .P 19 | When building \fBpackage\.yml\fR files (\fBypkg\fR), the tool will also disable all networking within the environment, apart from the loopback device\. This is intended to prevent uncontrolled build environments in which a package may be fetching external, unverified sources, during the build\. 20 | . 21 | .P 22 | This behaviour can be turned off on a package basis, by setting the \fBnetworking\fR key to \fBtrue\fR within the YML file\. This should only be used when it is completely unavoidable, however, as the container mechanism is there for a reason\. Trust\. 23 | . 24 | .P 25 | With both build types, legacy and \fBypkg\fR, the tool will enter an isolated namespace using the \fBunshare(2)\fR system call\. It intends to provide a highly controlled build environment, and providing a robust container in which to build packages intended for use in production\. 26 | . 27 | .SH "OPTIONS" 28 | These options apply to all subcommands within \fBsolbuild(1)\fR\. 29 | . 30 | .IP "\(bu" 4 31 | \fB\-h\fR, \fB\-\-help\fR 32 | . 33 | .IP 34 | Help provides an explanation for any command or subcommand\. Without any specified subcommands it will list the main subcommands for the application\. 35 | . 36 | .IP "\(bu" 4 37 | \fB\-n\fR, \fB\-\-no\-color\fR 38 | . 39 | .IP 40 | Disable text colourisation in the output from \fBsolbuild\fR and all child processes\. 41 | . 42 | .IP "\(bu" 4 43 | \fB\-p\fR, \fB\-\-profile\fR 44 | . 45 | .IP 46 | Set the build configuration profile to use with all operations\. 47 | . 48 | .IP "\(bu" 4 49 | \fB\-d\fR, \fB\-\-debug\fR 50 | . 51 | .IP 52 | Enable extra logging messages with debug level, useful to assist in further introspection of the environment setup and teardown\.\. 53 | . 54 | .IP "" 0 55 | . 56 | .SH "SUBCOMMANDS" 57 | \fBbuild [package\.yml] | [pspec\.xml]\fR 58 | . 59 | .IP "" 4 60 | . 61 | .nf 62 | 63 | Build the given package in a chroot environment, and upon success, 64 | store those packages in the current directory\. 65 | 66 | If you do not pass a package file as an argument to `build`, it will look 67 | for the files in the current working directory\. The priority is always given 68 | to `package\.yml` files, falling back to `pspec\.xml`, the legacy build format\. 69 | . 70 | .fi 71 | . 72 | .IP "" 0 73 | . 74 | .IP "\(bu" 4 75 | \fB\-t\fR, \fB\-\-tmpfs\fR: 76 | . 77 | .IP "" 4 78 | . 79 | .nf 80 | 81 | Instruct `solbuild(1)` to use a `tmpfs` mount as the bottom most point 82 | in the chroot layer system\. This can drastically improve build times, 83 | as most of the changes are happening purely in memory\. If running on 84 | a memory constrained device, please consider setting an appropriate 85 | upper constraint\. See the next flag for more details\. 86 | . 87 | .fi 88 | . 89 | .IP "" 0 90 | 91 | . 92 | .IP "\(bu" 4 93 | \fB\-m\fR, \fB\-\-memory\fR 94 | . 95 | .IP "" 4 96 | . 97 | .nf 98 | 99 | Set the contraint size for `tmpfs` mounts used by `solbuild(1)`\. This is 100 | only useful in conjunction with the `\-t` option\. 101 | . 102 | .fi 103 | . 104 | .IP "" 0 105 | 106 | . 107 | .IP "" 0 108 | . 109 | .P 110 | \fBchroot [package\.yml] | [pspec\.xml]\fR 111 | . 112 | .IP "" 4 113 | . 114 | .nf 115 | 116 | Interactively chroot into the package\'s build environment, to enable 117 | further inspection when issues aren\'t immediately resolvable, i\.e\. pkg\-config 118 | dependencies\. 119 | . 120 | .fi 121 | . 122 | .IP "" 0 123 | . 124 | .P 125 | \fBdelete\-cache\fR 126 | . 127 | .IP "" 4 128 | . 129 | .nf 130 | 131 | Delete all of the build roots under `/var/cache/solbuild`\. Although `solbuild(1)` 132 | employs many cache efficient methods in which to save on space and time, we 133 | retain the build roots after builds to allow inspection and chrooting\. 134 | 135 | Using this command will remove ALL roots from the cache\. You should ensure 136 | you are not already running any builds whilst calling this command, as it may 137 | lead to undefined behaviour\. 138 | . 139 | .fi 140 | . 141 | .IP "" 0 142 | . 143 | .IP "\(bu" 4 144 | \fB\-a\fR, \fB\-\-all\fR 145 | . 146 | .IP "" 4 147 | . 148 | .nf 149 | 150 | In addition to deleting the build root caches, the packages, sources, 151 | and ccache (compiler) caches will also be purged from disk\. 152 | . 153 | .fi 154 | . 155 | .IP "" 0 156 | 157 | . 158 | .IP "" 0 159 | . 160 | .P 161 | \fBindex [directory]\fR 162 | . 163 | .IP "" 4 164 | . 165 | .nf 166 | 167 | Use the given build profile to construct a repository index in the 168 | given directory\. If a directory is not specified, then the current directory 169 | is used\. This directory will be mounted inside the container and the Solus 170 | machinery will be used to create a repository\. 171 | . 172 | .fi 173 | . 174 | .IP "" 0 175 | . 176 | .IP "\(bu" 4 177 | \fB\-t\fR, \fB\-\-tmpfs\fR: 178 | . 179 | .IP "" 4 180 | . 181 | .nf 182 | 183 | Instruct `solbuild(1)` to use a `tmpfs` mount as the bottom most point 184 | in the chroot layer system\. This can help to speed up indexing of large 185 | repositories\. 186 | . 187 | .fi 188 | . 189 | .IP "" 0 190 | 191 | . 192 | .IP "\(bu" 4 193 | \fB\-m\fR, \fB\-\-memory\fR 194 | . 195 | .IP "" 4 196 | . 197 | .nf 198 | 199 | Set the contraint size for `tmpfs` mounts used by `solbuild(1)`\. This is 200 | only useful in conjunction with the `\-t` option\. 201 | . 202 | .fi 203 | . 204 | .IP "" 0 205 | 206 | . 207 | .IP "" 0 208 | . 209 | .P 210 | \fBinit\fR 211 | . 212 | .IP "" 4 213 | . 214 | .nf 215 | 216 | Initialise a solbuild profile so that it can be used for subsequent 217 | builds\. You must perform this step if you wish to do any kind of useful 218 | operations with `solbuild(1)`\. 219 | 220 | The init command respects the global `\-\-profile` option, however you 221 | may pass the name of the profile as an argument instead if you wish\. 222 | . 223 | .fi 224 | . 225 | .IP "" 0 226 | . 227 | .IP "\(bu" 4 228 | \fB\-u\fR, \fB\-\-update\fR 229 | . 230 | .IP "" 4 231 | . 232 | .nf 233 | 234 | Passing the update flag will cause `solbuild(1)` to automatically update 235 | the base image, after it has successfully initialised it\. 236 | . 237 | .fi 238 | . 239 | .IP "" 0 240 | 241 | . 242 | .IP "" 0 243 | . 244 | .P 245 | \fBupdate [profile]\fR 246 | . 247 | .IP "" 4 248 | . 249 | .nf 250 | 251 | Update the base image of the specified solbuild profile, helping to 252 | minimize the build times in future updates with this profile\. 253 | 254 | The update command respects the global `\-\-profile` option, however you 255 | may pass the name of the profile as an argument instead if you wish\. 256 | . 257 | .fi 258 | . 259 | .IP "" 0 260 | . 261 | .P 262 | \fBversion\fR 263 | . 264 | .IP "" 4 265 | . 266 | .nf 267 | 268 | Print the version and copyright notice of `solbuild(1)` and exit\. 269 | . 270 | .fi 271 | . 272 | .IP "" 0 273 | . 274 | .SH "EXIT STATUS" 275 | On success, 0 is returned\. A non\-zero return code signals a failure\. 276 | . 277 | .SH "COPYRIGHT" 278 | . 279 | .IP "\(bu" 4 280 | Copyright © 2016 Ikey Doherty, License: CC\-BY\-SA\-3\.0 281 | . 282 | .IP "" 0 283 | . 284 | .SH "SEE ALSO" 285 | \fBsolbuild\.conf(5)\fR, \fBsolbuild\.profile(5)\fR 286 | . 287 | .P 288 | https://github\.com/solus\-project/solbuild 289 | . 290 | .P 291 | https://github\.com/solus\-project/ypkg 292 | . 293 | .SH "NOTES" 294 | Creative Commons Attribution\-ShareAlike 3\.0 Unported 295 | . 296 | .IP "\(bu" 4 297 | http://creativecommons\.org/licenses/by\-sa/3\.0/ 298 | . 299 | .IP "" 0 300 | 301 | -------------------------------------------------------------------------------- /man/solbuild.1.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | solbuild(1) - Solus package builder 7 | 44 | 45 | 52 | 53 |
54 | 55 | 66 | 67 |
    68 |
  1. solbuild(1)
    2. 69 |
  2. 70 |
  3. solbuild(1)
    4. 71 |
72 | 73 |

NAME

74 |

75 | solbuild - Solus package builder 76 |

77 | 78 |

SYNOPSIS

79 | 80 |

solbuild [subcommand] <flags>

81 | 82 |

DESCRIPTION

83 | 84 |

solbuild(1) is a chroot(2) based package build system, used to safely and 85 | efficiently build Solus packages from source.

86 | 87 |

solbuild(1) makes use of OverlayFS to provide a simple caching system, whereby 88 | a base image (provided by the Solus project) is used as the bottom-most, read-only 89 | layer, and changes are made in temporary upper layers.

90 | 91 |

When building package.yml files (ypkg), the tool will also disable all 92 | networking within the environment, apart from the loopback device. This is 93 | intended to prevent uncontrolled build environments in which a package may 94 | be fetching external, unverified sources, during the build.

95 | 96 |

This behaviour can be turned off on a package basis, by setting the networking 97 | key to true within the YML file. This should only be used when it is completely 98 | unavoidable, however, as the container mechanism is there for a reason. Trust.

99 | 100 |

With both build types, legacy and ypkg, the tool will enter an isolated namespace 101 | using the unshare(2) system call. It intends to provide a highly controlled 102 | build environment, and providing a robust container in which to build packages 103 | intended for use in production.

104 | 105 |

OPTIONS

106 | 107 |

These options apply to all subcommands within solbuild(1).

108 | 109 | 126 | 127 | 128 |

SUBCOMMANDS

129 | 130 |

build [package.yml] | [pspec.xml]

131 | 132 | 
Build the given package in a chroot environment, and upon success,
133 | store those packages in the current directory.
134 | 
135 | If you do not pass a package file as an argument to `build`, it will look
136 | for the files in the current working directory. The priority is always given
137 | to `package.yml` files, falling back to `pspec.xml`, the legacy build format.
138 |
139 | 140 | 155 | 156 | 157 |

chroot [package.yml] | [pspec.xml]

158 | 159 | 
Interactively chroot into the package's build environment, to enable
160 | further inspection when issues aren't immediately resolvable, i.e. pkg-config
161 | dependencies.
162 |
163 | 164 |

delete-cache

165 | 166 | 
Delete all of the build roots under `/var/cache/solbuild`. Although `solbuild(1)`
167 | employs many cache efficient methods in which to save on space and time, we
168 | retain the build roots after builds to allow inspection and chrooting.
169 | 
170 | Using this command will remove ALL roots from the cache. You should ensure
171 | you are not already running any builds whilst calling this command, as it may
172 | lead to undefined behaviour.
173 |
174 | 175 | 182 | 183 | 184 |

index [directory]

185 | 186 | 
Use the given build profile to construct a repository index in the
187 | given directory. If a directory is not specified, then the current directory
188 | is used. This directory will be mounted inside the container and the Solus
189 | machinery will be used to create a repository.
190 |
191 | 192 | 205 | 206 | 207 |

init

208 | 209 | 
Initialise a solbuild profile so that it can be used for subsequent
210 | builds. You must perform this step if you wish to do any kind of useful
211 | operations with `solbuild(1)`.
212 | 
213 | The init command respects the global `--profile` option, however you
214 | may pass the name of the profile as an argument instead if you wish.
215 |
216 | 217 | 224 | 225 | 226 |

update [profile]

227 | 228 | 
Update the base image of the specified solbuild profile, helping to
229 | minimize the build times in future updates with this profile.
230 | 
231 | The update command respects the global `--profile` option, however you
232 | may pass the name of the profile as an argument instead if you wish.
233 |
234 | 235 |

version

236 | 237 | 
Print the version and copyright notice of `solbuild(1)` and exit.
238 |
239 | 240 |

EXIT STATUS

241 | 242 |

On success, 0 is returned. A non-zero return code signals a failure.

243 | 244 | 245 | 246 | 249 | 250 | 251 |

SEE ALSO

252 | 253 |

solbuild.conf(5), solbuild.profile(5)

254 | 255 |

https://github.com/solus-project/solbuild

256 | 257 |

https://github.com/solus-project/ypkg

258 | 259 |

NOTES

260 | 261 |

Creative Commons Attribution-ShareAlike 3.0 Unported

262 | 263 | 266 | 267 | 268 | 269 |
    270 |
  1. 271 |
  2. January 2017
    3. 272 |
  3. solbuild(1)
    4. 273 |
274 | 275 |
276 | 277 | 278 | -------------------------------------------------------------------------------- /man/solbuild.1.md: -------------------------------------------------------------------------------- 1 | solbuild(1) -- Solus package builder 2 | ================================= 3 | 4 | 5 | ## SYNOPSIS 6 | 7 | `solbuild [subcommand] ` 8 | 9 | 10 | ## DESCRIPTION 11 | 12 | `solbuild(1)` is a `chroot(2)` based package build system, used to safely and 13 | efficiently build Solus packages from source. 14 | 15 | `solbuild(1)` makes use of `OverlayFS` to provide a simple caching system, whereby 16 | a base image (provided by the Solus project) is used as the bottom-most, read-only 17 | layer, and changes are made in temporary upper layers. 18 | 19 | When building `package.yml` files (`ypkg`), the tool will also disable all 20 | networking within the environment, apart from the loopback device. This is 21 | intended to prevent uncontrolled build environments in which a package may 22 | be fetching external, unverified sources, during the build. 23 | 24 | This behaviour can be turned off on a package basis, by setting the `networking` 25 | key to `true` within the YML file. This should only be used when it is completely 26 | unavoidable, however, as the container mechanism is there for a reason. Trust. 27 | 28 | With both build types, legacy and `ypkg`, the tool will enter an isolated namespace 29 | using the `unshare(2)` system call. It intends to provide a highly controlled 30 | build environment, and providing a robust container in which to build packages 31 | intended for use in production. 32 | 33 | ## OPTIONS 34 | 35 | These options apply to all subcommands within `solbuild(1)`. 36 | 37 | * `-h`, `--help` 38 | 39 | Help provides an explanation for any command or subcommand. Without any 40 | specified subcommands it will list the main subcommands for the application. 41 | 42 | * `-n`, `--no-color` 43 | 44 | Disable text colourisation in the output from `solbuild` and all child 45 | processes. 46 | 47 | * `-p`, `--profile` 48 | 49 | Set the build configuration profile to use with all operations. 50 | 51 | * `-d`, `--debug` 52 | 53 | Enable extra logging messages with debug level, useful to assist in further 54 | introspection of the environment setup and teardown.. 55 | 56 | 57 | ## SUBCOMMANDS 58 | 59 | 60 | `build [package.yml] | [pspec.xml]` 61 | 62 | Build the given package in a chroot environment, and upon success, 63 | store those packages in the current directory. 64 | 65 | If you do not pass a package file as an argument to `build`, it will look 66 | for the files in the current working directory. The priority is always given 67 | to `package.yml` files, falling back to `pspec.xml`, the legacy build format. 68 | 69 | * `-t`, `--tmpfs`: 70 | 71 | Instruct `solbuild(1)` to use a `tmpfs` mount as the bottom most point 72 | in the chroot layer system. This can drastically improve build times, 73 | as most of the changes are happening purely in memory. If running on 74 | a memory constrained device, please consider setting an appropriate 75 | upper constraint. See the next flag for more details. 76 | 77 | * `-m`, `--memory` 78 | 79 | Set the contraint size for `tmpfs` mounts used by `solbuild(1)`. This is 80 | only useful in conjunction with the `-t` option. 81 | 82 | `chroot [package.yml] | [pspec.xml]` 83 | 84 | Interactively chroot into the package's build environment, to enable 85 | further inspection when issues aren't immediately resolvable, i.e. pkg-config 86 | dependencies. 87 | 88 | `delete-cache` 89 | 90 | Delete all of the build roots under `/var/cache/solbuild`. Although `solbuild(1)` 91 | employs many cache efficient methods in which to save on space and time, we 92 | retain the build roots after builds to allow inspection and chrooting. 93 | 94 | Using this command will remove ALL roots from the cache. You should ensure 95 | you are not already running any builds whilst calling this command, as it may 96 | lead to undefined behaviour. 97 | 98 | * `-a`, `--all` 99 | 100 | In addition to deleting the build root caches, the packages, sources, 101 | and ccache (compiler) caches will also be purged from disk. 102 | 103 | `index [directory]` 104 | 105 | Use the given build profile to construct a repository index in the 106 | given directory. If a directory is not specified, then the current directory 107 | is used. This directory will be mounted inside the container and the Solus 108 | machinery will be used to create a repository. 109 | 110 | 111 | * `-t`, `--tmpfs`: 112 | 113 | Instruct `solbuild(1)` to use a `tmpfs` mount as the bottom most point 114 | in the chroot layer system. This can help to speed up indexing of large 115 | repositories. 116 | 117 | * `-m`, `--memory` 118 | 119 | Set the contraint size for `tmpfs` mounts used by `solbuild(1)`. This is 120 | only useful in conjunction with the `-t` option. 121 | 122 | `init` 123 | 124 | Initialise a solbuild profile so that it can be used for subsequent 125 | builds. You must perform this step if you wish to do any kind of useful 126 | operations with `solbuild(1)`. 127 | 128 | The init command respects the global `--profile` option, however you 129 | may pass the name of the profile as an argument instead if you wish. 130 | 131 | * `-u`, `--update` 132 | 133 | Passing the update flag will cause `solbuild(1)` to automatically update 134 | the base image, after it has successfully initialised it. 135 | 136 | `update [profile]` 137 | 138 | Update the base image of the specified solbuild profile, helping to 139 | minimize the build times in future updates with this profile. 140 | 141 | The update command respects the global `--profile` option, however you 142 | may pass the name of the profile as an argument instead if you wish. 143 | 144 | `version` 145 | 146 | Print the version and copyright notice of `solbuild(1)` and exit. 147 | 148 | 149 | ## EXIT STATUS 150 | 151 | On success, 0 is returned. A non-zero return code signals a failure. 152 | 153 | 154 | ## COPYRIGHT 155 | 156 | * Copyright © 2016 Ikey Doherty, License: CC-BY-SA-3.0 157 | 158 | 159 | ## SEE ALSO 160 | 161 | `solbuild.conf(5)`, `solbuild.profile(5)` 162 | 163 | https://github.com/solus-project/solbuild 164 | 165 | https://github.com/solus-project/ypkg 166 | 167 | 168 | ## NOTES 169 | 170 | Creative Commons Attribution-ShareAlike 3.0 Unported 171 | 172 | * http://creativecommons.org/licenses/by-sa/3.0/ 173 | -------------------------------------------------------------------------------- /man/solbuild.conf.5: -------------------------------------------------------------------------------- 1 | .\" generated with Ronn/v0.7.3 2 | .\" http://github.com/rtomayko/ronn/tree/0.7.3 3 | . 4 | .TH "SOLBUILD\.CONF" "5" "January 2017" "" "" 5 | . 6 | .SH "NAME" 7 | \fBsolbuild\.conf\fR \- solbuild configuration 8 | . 9 | .SH "NAME" 10 | . 11 | .nf 12 | 13 | solbuild\.conf \- configuration for solbuild 14 | . 15 | .fi 16 | . 17 | .SH "SYNOPSIS" 18 | . 19 | .nf 20 | 21 | /usr/share/solbuild/*\.conf 22 | 23 | /etc/solbuild/*\.conf 24 | . 25 | .fi 26 | . 27 | .SH "DESCRIPTION" 28 | \fBsolbuild(1)\fR uses configuration files from the above mentioned directories to configure various aspects of the \fBsolbuild\fR defaults\. 29 | . 30 | .P 31 | All configuration files must be valid prior to \fBsolbuild(1)\fR launching, as it will load and validate them all into a merged configuration\. Using a layered approach, \fBsolbuild\fR will first read from the global vendor directory, \fB/usr/share/solbuild\fR, before finally loading from the system directory, \fB/etc/solbuild\fR\. 32 | . 33 | .P 34 | \fBsolbuild(1)\fR is capable of running without configuration, and this method permits a stateless implementation whereby vendor & system administrator configurations are respected in the correct order\. 35 | . 36 | .SH "CONFIGURATION FORMAT" 37 | \fBsolbuild\fR uses the \fBTOML\fR configuration format for all of it\'s own configuration files\. This is a strongly typed configuration format, whereby strict validation occurs against expected key types\. 38 | . 39 | .IP "\(bu" 4 40 | \fBdefault_profile\fR 41 | . 42 | .IP 43 | Set the default profile used by \fBsolbuild(1)\fR\. This must have a string value, and will be used by \fBsolbuild(1)\fR in the absence of the \fB\-p\fR,\fB\-\-profile\fR flag\. 44 | . 45 | .IP "\(bu" 4 46 | \fBenable_tmpfs\fR 47 | . 48 | .IP 49 | Instruct \fBsolbuild(1)\fR to use tmpfs mounts by default for all builds\. Note that even if this is disabled, as it is by default, you may still override this at runtime with the \fB\-t\fR,\fB\-\-tmpfs\fR flag\. 50 | . 51 | .IP "\(bu" 4 52 | \fBtmpfs_size\fR 53 | . 54 | .IP 55 | Set the default tmpfs size used by \fBsolbuild(1)\fR when tmpfs builds are enabled\. An empty value, the default, will mean an unbounded size to the tmpfs\. This value should be a string value, with the same syntax that one would pass to \fBmount(8)\fR\. 56 | . 57 | .IP 58 | See \fBsolbuild(1)\fR for more details on the \fB\-t\fR,\fB\-\-tmpfs\fR option behaviour\. 59 | . 60 | .IP "" 0 61 | . 62 | .SH "EXAMPLE" 63 | . 64 | .nf 65 | 66 | # Set the default profile, a string value assignment 67 | default_profile = "main\-x86_64" 68 | 69 | # Set tmpfs enabled by default, a boolean value assignment 70 | enable_tmpfs = true 71 | . 72 | .fi 73 | . 74 | .SH "COPYRIGHT" 75 | . 76 | .IP "\(bu" 4 77 | Copyright © 2016 Ikey Doherty, License: CC\-BY\-SA\-3\.0 78 | . 79 | .IP "" 0 80 | . 81 | .SH "SEE ALSO" 82 | \fBsolbuild(1)\fR, \fBsolbuild\.profile(5)\fR 83 | . 84 | .P 85 | https://github\.com/toml\-lang/toml 86 | . 87 | .SH "NOTES" 88 | Creative Commons Attribution\-ShareAlike 3\.0 Unported 89 | . 90 | .IP "\(bu" 4 91 | http://creativecommons\.org/licenses/by\-sa/3\.0/ 92 | . 93 | .IP "" 0 94 | 95 | -------------------------------------------------------------------------------- /man/solbuild.conf.5.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | solbuild.conf(5) - solbuild configuration 7 | 44 | 45 | 52 | 53 |
54 | 55 | 66 | 67 |
    68 |
  1. solbuild.conf(5)
    2. 69 |
  2. 70 |
  3. solbuild.conf(5)
    4. 71 |
72 | 73 |

NAME

74 |

75 | solbuild.conf - solbuild configuration 76 |

77 | 78 |

NAME

79 | 80 | 
solbuild.conf - configuration for solbuild
 81 |
82 | 83 |

SYNOPSIS

84 | 85 | 
/usr/share/solbuild/*.conf
 86 | 
 87 | /etc/solbuild/*.conf
 88 |
89 | 90 |

DESCRIPTION

91 | 92 |

solbuild(1) uses configuration files from the above mentioned directories to 93 | configure various aspects of the solbuild defaults.

94 | 95 |

All configuration files must be valid prior to solbuild(1) launching, as it 96 | will load and validate them all into a merged configuration. Using a layered 97 | approach, solbuild will first read from the global vendor directory, 98 | /usr/share/solbuild, before finally loading from the system directory, 99 | /etc/solbuild.

100 | 101 |

solbuild(1) is capable of running without configuration, and this method 102 | permits a stateless implementation whereby vendor & system administrator 103 | configurations are respected in the correct order.

104 | 105 |

CONFIGURATION FORMAT

106 | 107 |

solbuild uses the TOML configuration format for all of it's own 108 | configuration files. This is a strongly typed configuration format, whereby 109 | strict validation occurs against expected key types.

110 | 111 |
    112 |

  • default_profile

    113 | 114 |

    Set the default profile used by solbuild(1). This must have a string value, 115 | and will be used by solbuild(1) in the absence of the -p,--profile 116 | flag.

    • 117 |

  • enable_tmpfs

    118 | 119 |

    Instruct solbuild(1) to use tmpfs mounts by default for all builds. Note 120 | that even if this is disabled, as it is by default, you may still override 121 | this at runtime with the -t,--tmpfs flag.

    • 122 |

  • tmpfs_size

    123 | 124 |

    Set the default tmpfs size used by solbuild(1) when tmpfs builds are 125 | enabled. An empty value, the default, will mean an unbounded size to 126 | the tmpfs. This value should be a string value, with the same syntax 127 | that one would pass to mount(8).

    128 | 129 |

    See solbuild(1) for more details on the -t,--tmpfs option behaviour.

    • 130 |
131 | 132 | 133 |

EXAMPLE

134 | 135 | 
# Set the default profile, a string value assignment
136 | default_profile = "main-x86_64"
137 | 
138 | # Set tmpfs enabled by default, a boolean value assignment
139 | enable_tmpfs = true
140 |
141 | 142 | 143 | 144 |
    145 |
  • Copyright © 2016 Ikey Doherty, License: CC-BY-SA-3.0
    • 146 |
147 | 148 | 149 |

SEE ALSO

150 | 151 |

solbuild(1), solbuild.profile(5)

152 | 153 |

https://github.com/toml-lang/toml

154 | 155 |

NOTES

156 | 157 |

Creative Commons Attribution-ShareAlike 3.0 Unported

158 | 159 |
    160 |
  • http://creativecommons.org/licenses/by-sa/3.0/
    • 161 |
162 | 163 | 164 | 165 |
    166 |
  1. 167 |
  2. January 2017
    3. 168 |
  3. solbuild.conf(5)
    4. 169 |
170 | 171 |
172 | 173 | 174 | -------------------------------------------------------------------------------- /man/solbuild.conf.5.md: -------------------------------------------------------------------------------- 1 | solbuild.conf(5) -- solbuild configuration 2 | ========================================== 3 | 4 | ## NAME 5 | 6 | solbuild.conf - configuration for solbuild 7 | 8 | ## SYNOPSIS 9 | 10 | /usr/share/solbuild/*.conf 11 | 12 | /etc/solbuild/*.conf 13 | 14 | 15 | ## DESCRIPTION 16 | 17 | `solbuild(1)` uses configuration files from the above mentioned directories to 18 | configure various aspects of the `solbuild` defaults. 19 | 20 | All configuration files must be valid prior to `solbuild(1)` launching, as it 21 | will load and validate them all into a merged configuration. Using a layered 22 | approach, `solbuild` will first read from the global vendor directory, 23 | `/usr/share/solbuild`, before finally loading from the system directory, 24 | `/etc/solbuild`. 25 | 26 | `solbuild(1)` is capable of running without configuration, and this method 27 | permits a stateless implementation whereby vendor & system administrator 28 | configurations are respected in the correct order. 29 | 30 | ## CONFIGURATION FORMAT 31 | 32 | `solbuild` uses the `TOML` configuration format for all of it's own 33 | configuration files. This is a strongly typed configuration format, whereby 34 | strict validation occurs against expected key types. 35 | 36 | * `default_profile` 37 | 38 | Set the default profile used by `solbuild(1)`. This must have a string value, 39 | and will be used by `solbuild(1)` in the absence of the `-p`,`--profile` 40 | flag. 41 | 42 | * `enable_tmpfs` 43 | 44 | Instruct `solbuild(1)` to use tmpfs mounts by default for all builds. Note 45 | that even if this is disabled, as it is by default, you may still override 46 | this at runtime with the `-t`,`--tmpfs` flag. 47 | 48 | * `tmpfs_size` 49 | 50 | Set the default tmpfs size used by `solbuild(1)` when tmpfs builds are 51 | enabled. An empty value, the default, will mean an unbounded size to 52 | the tmpfs. This value should be a string value, with the same syntax 53 | that one would pass to `mount(8)`. 54 | 55 | See `solbuild(1)` for more details on the `-t`,`--tmpfs` option behaviour. 56 | 57 | 58 | ## EXAMPLE 59 | 60 | # Set the default profile, a string value assignment 61 | default_profile = "main-x86_64" 62 | 63 | # Set tmpfs enabled by default, a boolean value assignment 64 | enable_tmpfs = true 65 | 66 | 67 | ## COPYRIGHT 68 | 69 | * Copyright © 2016 Ikey Doherty, License: CC-BY-SA-3.0 70 | 71 | 72 | ## SEE ALSO 73 | 74 | 75 | `solbuild(1)`, `solbuild.profile(5)` 76 | 77 | https://github.com/toml-lang/toml 78 | 79 | ## NOTES 80 | 81 | Creative Commons Attribution-ShareAlike 3.0 Unported 82 | 83 | * http://creativecommons.org/licenses/by-sa/3.0/ 84 | -------------------------------------------------------------------------------- /man/solbuild.profile.5: -------------------------------------------------------------------------------- 1 | .\" generated with Ronn/v0.7.3 2 | .\" http://github.com/rtomayko/ronn/tree/0.7.3 3 | . 4 | .TH "SOLBUILD\.PROFILE" "5" "January 2017" "" "" 5 | . 6 | .SH "NAME" 7 | \fBsolbuild\.profile\fR \- Profile definitions for solbuild 8 | . 9 | .SH "SYNOPSIS" 10 | . 11 | .nf 12 | 13 | /usr/share/solbuild/*\.profile 14 | 15 | /etc/solbuild/*\.profile 16 | . 17 | .fi 18 | . 19 | .SH "DESCRIPTION" 20 | \fBsolbuild(1)\fR uses configuration files from the above mentioned directories to define profiles used for builds\. A \fBsolbuild\fR profile is automatically named to the basename of the file, without the \fB\.profile\fR suffix\. 21 | . 22 | .P 23 | As an example, if we have the file \fB/etc/solbuild/test\.profile\fR, the name of the profile in \fBsolbuild(1)\fR would be \fBtest\fR\. With the layered stateless approach in solbuild, any named profile in the system config directory \fB/etc/\fR will take priority over the named profiles in the vendor directory\. These profiles are not merged, the one in \fB/etc/\fR will "replace" the one in the vendor directory, \fB/usr/share/solbuild\fR\. 24 | . 25 | .SH "CONFIGURATION FORMAT" 26 | \fBsolbuild\fR uses the \fBTOML\fR configuration format for all of it\'s own configuration files\. This is a strongly typed configuration format, whereby strict validation occurs against expected key types\. 27 | . 28 | .IP "\(bu" 4 29 | \fBimage\fR 30 | . 31 | .IP 32 | Set the backing image to one of the (currently Solus) provided backing images\. Valid values include: 33 | . 34 | .IP "" 4 35 | . 36 | .nf 37 | 38 | * `main\-x86_64` 39 | * `unstable\-x86_64` 40 | . 41 | .fi 42 | . 43 | .IP "" 0 44 | . 45 | .IP 46 | A string value is expected for this key\. 47 | . 48 | .IP "\(bu" 4 49 | \fBremove_repos\fR 50 | . 51 | .IP 52 | This key expects an array of strings for the repo names to remove from the existing base image during builds\. Currently the Solus provided images all use the name \fBSolus\fR for the repo they ship with\. 53 | . 54 | .IP 55 | Setting this to a value of \fB[\'*\']\fR will indicate removal of all repos\. 56 | . 57 | .IP "\(bu" 4 58 | \fBadd_repos\fR 59 | . 60 | .IP 61 | This key expects an array of strings for the repo names defined in this profile to add to the image\. The default unset value, i\.e\. an absence of this key, or the special value \fB[\'*\']\fR will enable all of the repos in the profile\. 62 | . 63 | .IP 64 | This option may be useful for testing repos and conditionally disabling them for testing, without having to remove them from the file\. 65 | . 66 | .IP "\(bu" 4 67 | \fB[repo\.$Name]\fR 68 | . 69 | .IP 70 | A repository is defined with this key, where \fB$Name\fR is replaced with the name you intend to assign to the repository\. By default, a repo definition is configured for a remote repository\. 71 | . 72 | .IP "\(bu" 4 73 | \fB[repo\.$Name]\fR \fBuri\fR 74 | . 75 | .IP 76 | Set this to the remote repository URL, including the \fBeopkg\-index\.xml\.xz\fR If the repository is a \fBlocal\fR one, you must include the path to the directory, with no suffix\. 77 | . 78 | .IP "\(bu" 4 79 | \fB[repo\.$Name]\fR \fBlocal\fR 80 | . 81 | .IP 82 | Set this to true to configure \fBsolbuild(1)\fR to add a local repository to the build\. The build process will bind\-mount the \fBuri\fR configured directory into the build and make it available\. 83 | . 84 | .IP "\(bu" 4 85 | \fB[repo\.$Name]\fR \fBautoindex\fR 86 | . 87 | .IP 88 | Set this to true to instruct \fBsolbuild(1)\fR to automatically reindex this local repository while in the container\. This may be useful if you do not have the appropriate host side tools\. 89 | . 90 | .IP 91 | \fBsolbuild(1)\fR will only index the files once, at startup, before it has performed the upgrade and component validation\. Once your build has completed, and your \fB*\.eopkg\fR files are deposited in your current directory, you can simply copy them to your local repository directory, and then \fBsolbuild\fR will be able to use them immediately in your next build\. 92 | . 93 | .IP "" 0 94 | 95 | . 96 | .IP "" 0 97 | . 98 | .SH "EXAMPLE" 99 | . 100 | .nf 101 | 102 | # Use the unstable backing image for this profile 103 | image = "unstable\-x86_64" 104 | 105 | # Restrict adding the repos to the Solus repo only 106 | add_repos = [\'Solus\'] 107 | 108 | # Example of adding a remote repo 109 | [repo\.Solus] 110 | uri = "https://packages\.solus\-project\.com/unstable/eopkg\-index\.xml\.xz" 111 | 112 | # Add a local repository by bind mounting it into chroot on each build 113 | [repo\.Local] 114 | uri = "/var/lib/myrepo" 115 | local = true 116 | 117 | # If you have a local repo providing packages that exist in the main 118 | # repository already, you should remove the repo, and re\-add it *after* 119 | # your local repository: 120 | remove_repos = [\'Solus\'] 121 | add_repos = [\'Local\',\'Solus\'] 122 | . 123 | .fi 124 | . 125 | .SH "COPYRIGHT" 126 | . 127 | .IP "\(bu" 4 128 | Copyright © 2016 Ikey Doherty, License: CC\-BY\-SA\-3\.0 129 | . 130 | .IP "" 0 131 | . 132 | .SH "SEE ALSO" 133 | \fBsolbuild(1)\fR, \fBsolbuild\.conf(5)\fR 134 | . 135 | .P 136 | https://github\.com/toml\-lang/toml 137 | . 138 | .SH "NOTES" 139 | Creative Commons Attribution\-ShareAlike 3\.0 Unported 140 | . 141 | .IP "\(bu" 4 142 | http://creativecommons\.org/licenses/by\-sa/3\.0/ 143 | . 144 | .IP "" 0 145 | 146 | -------------------------------------------------------------------------------- /man/solbuild.profile.5.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | solbuild.profile(5) - Profile definitions for solbuild 7 | 44 | 45 | 52 | 53 |
54 | 55 | 65 | 66 |
    67 |
  1. solbuild.profile(5)
    2. 68 |
  2. 69 |
  3. solbuild.profile(5)
    4. 70 |
71 | 72 |

NAME

73 |

74 | solbuild.profile - Profile definitions for solbuild 75 |

76 | 77 |

SYNOPSIS

78 | 79 | 
/usr/share/solbuild/*.profile
 80 | 
 81 | /etc/solbuild/*.profile
 82 |
83 | 84 |

DESCRIPTION

85 | 86 |

solbuild(1) uses configuration files from the above mentioned directories to 87 | define profiles used for builds. A solbuild profile is automatically named 88 | to the basename of the file, without the .profile suffix.

89 | 90 |

As an example, if we have the file /etc/solbuild/test.profile, the name of 91 | the profile in solbuild(1) would be test. With the layered stateless 92 | approach in solbuild, any named profile in the system config directory /etc/ 93 | will take priority over the named profiles in the vendor directory. These 94 | profiles are not merged, the one in /etc/ will "replace" the one in the 95 | vendor directory, /usr/share/solbuild.

96 | 97 |

CONFIGURATION FORMAT

98 | 99 |

solbuild uses the TOML configuration format for all of it's own 100 | configuration files. This is a strongly typed configuration format, whereby 101 | strict validation occurs against expected key types.

102 | 103 |
    104 |

  • image

    105 | 106 |

    Set the backing image to one of the (currently Solus) provided backing 107 | images. Valid values include:

    108 | 109 | 
      * `main-x86_64`
110 |   * `unstable-x86_64`
111 |
    112 | 113 |

    A string value is expected for this key.

    • 114 |

  • remove_repos

    115 | 116 |

    This key expects an array of strings for the repo names to remove from the 117 | existing base image during builds. Currently the Solus provided images all 118 | use the name Solus for the repo they ship with.

    119 | 120 |

    Setting this to a value of ['*'] will indicate removal of all repos.

    • 121 |

  • add_repos

    122 | 123 |

    This key expects an array of strings for the repo names defined in this 124 | profile to add to the image. The default unset value, i.e. an absence 125 | of this key, or the special value ['*'] will enable all of the repos 126 | in the profile.

    127 | 128 |

    This option may be useful for testing repos and conditionally disabling 129 | them for testing, without having to remove them from the file.

    • 130 |

  • [repo.$Name]

    131 | 132 |

    A repository is defined with this key, where $Name is replaced with the 133 | name you intend to assign to the repository. By default, a repo definition 134 | is configured for a remote repository.

    135 | 136 |
      137 |

    • [repo.$Name] uri

      138 | 139 |

      Set this to the remote repository URL, including the eopkg-index.xml.xz 140 | If the repository is a local one, you must include the path to the 141 | directory, with no suffix.

      • 142 |

    • [repo.$Name] local

      143 | 144 |

      Set this to true to configure solbuild(1) to add a local repository 145 | to the build. The build process will bind-mount the uri configured 146 | directory into the build and make it available.

      • 147 |

    • [repo.$Name] autoindex

      148 | 149 |

      Set this to true to instruct solbuild(1) to automatically reindex this 150 | local repository while in the container. This may be useful if you do 151 | not have the appropriate host side tools.

      152 | 153 |

      solbuild(1) will only index the files once, at startup, before it has 154 | performed the upgrade and component validation. Once your build has 155 | completed, and your *.eopkg files are deposited in your current directory, 156 | you can simply copy them to your local repository directory, and then 157 | solbuild will be able to use them immediately in your next build.

      • 158 |
    159 |
    • 160 |
161 | 162 | 163 |

EXAMPLE

164 | 165 | 
# Use the unstable backing image for this profile
166 | image = "unstable-x86_64"
167 | 
168 | # Restrict adding the repos to the Solus repo only
169 | add_repos = ['Solus']
170 | 
171 | # Example of adding a remote repo
172 | [repo.Solus]
173 | uri = "https://packages.solus-project.com/unstable/eopkg-index.xml.xz"
174 | 
175 | # Add a local repository by bind mounting it into chroot on each build
176 | [repo.Local]
177 | uri = "/var/lib/myrepo"
178 | local = true
179 | 
180 | # If you have a local repo providing packages that exist in the main
181 | # repository already, you should remove the repo, and re-add it *after*
182 | # your local repository:
183 | remove_repos = ['Solus']
184 | add_repos = ['Local','Solus']
185 |
186 | 187 | 188 | 189 |
    190 |
  • Copyright © 2016 Ikey Doherty, License: CC-BY-SA-3.0
    • 191 |
192 | 193 | 194 |

SEE ALSO

195 | 196 |

solbuild(1), solbuild.conf(5)

197 | 198 |

https://github.com/toml-lang/toml

199 | 200 |

NOTES

201 | 202 |

Creative Commons Attribution-ShareAlike 3.0 Unported

203 | 204 |
    205 |
  • http://creativecommons.org/licenses/by-sa/3.0/
    • 206 |
207 | 208 | 209 | 210 |
    211 |
  1. 212 |
  2. January 2017
    3. 213 |
  3. solbuild.profile(5)
    4. 214 |
215 | 216 |
217 | 218 | 219 | -------------------------------------------------------------------------------- /man/solbuild.profile.5.md: -------------------------------------------------------------------------------- 1 | solbuild.profile(5) -- Profile definitions for solbuild 2 | ======================================================= 3 | 4 | ## SYNOPSIS 5 | 6 | /usr/share/solbuild/*.profile 7 | 8 | /etc/solbuild/*.profile 9 | 10 | 11 | ## DESCRIPTION 12 | 13 | `solbuild(1)` uses configuration files from the above mentioned directories to 14 | define profiles used for builds. A `solbuild` profile is automatically named 15 | to the basename of the file, without the `.profile` suffix. 16 | 17 | As an example, if we have the file `/etc/solbuild/test.profile`, the name of 18 | the profile in `solbuild(1)` would be **test**. With the layered stateless 19 | approach in solbuild, any named profile in the system config directory `/etc/` 20 | will take priority over the named profiles in the vendor directory. These 21 | profiles are not merged, the one in `/etc/` will "replace" the one in the 22 | vendor directory, `/usr/share/solbuild`. 23 | 24 | 25 | ## CONFIGURATION FORMAT 26 | 27 | `solbuild` uses the `TOML` configuration format for all of it's own 28 | configuration files. This is a strongly typed configuration format, whereby 29 | strict validation occurs against expected key types. 30 | 31 | * `image` 32 | 33 | Set the backing image to one of the (currently Solus) provided backing 34 | images. Valid values include: 35 | 36 | * `main-x86_64` 37 | * `unstable-x86_64` 38 | 39 | A string value is expected for this key. 40 | 41 | * `remove_repos` 42 | 43 | This key expects an array of strings for the repo names to remove from the 44 | existing base image during builds. Currently the Solus provided images all 45 | use the name **Solus** for the repo they ship with. 46 | 47 | Setting this to a value of `['*']` will indicate removal of all repos. 48 | 49 | * `add_repos` 50 | 51 | This key expects an array of strings for the repo names defined in this 52 | profile to add to the image. The default unset value, i.e. an absence 53 | of this key, or the special value `['*']` will enable all of the repos 54 | in the profile. 55 | 56 | This option may be useful for testing repos and conditionally disabling 57 | them for testing, without having to remove them from the file. 58 | 59 | * `[repo.$Name]` 60 | 61 | A repository is defined with this key, where `$Name` is replaced with the 62 | name you intend to assign to the repository. By default, a repo definition 63 | is configured for a remote repository. 64 | 65 | * `[repo.$Name]` `uri` 66 | 67 | Set this to the remote repository URL, including the `eopkg-index.xml.xz` 68 | If the repository is a **local** one, you must include the path to the 69 | directory, with no suffix. 70 | 71 | * `[repo.$Name]` `local` 72 | 73 | Set this to true to configure `solbuild(1)` to add a local repository 74 | to the build. The build process will bind-mount the `uri` configured 75 | directory into the build and make it available. 76 | 77 | * `[repo.$Name]` `autoindex` 78 | 79 | Set this to true to instruct `solbuild(1)` to automatically reindex this 80 | local repository while in the container. This may be useful if you do 81 | not have the appropriate host side tools. 82 | 83 | `solbuild(1)` will only index the files once, at startup, before it has 84 | performed the upgrade and component validation. Once your build has 85 | completed, and your `*.eopkg` files are deposited in your current directory, 86 | you can simply copy them to your local repository directory, and then 87 | `solbuild` will be able to use them immediately in your next build. 88 | 89 | 90 | ## EXAMPLE 91 | 92 | # Use the unstable backing image for this profile 93 | image = "unstable-x86_64" 94 | 95 | # Restrict adding the repos to the Solus repo only 96 | add_repos = ['Solus'] 97 | 98 | # Example of adding a remote repo 99 | [repo.Solus] 100 | uri = "https://packages.solus-project.com/unstable/eopkg-index.xml.xz" 101 | 102 | # Add a local repository by bind mounting it into chroot on each build 103 | [repo.Local] 104 | uri = "/var/lib/myrepo" 105 | local = true 106 | 107 | # If you have a local repo providing packages that exist in the main 108 | # repository already, you should remove the repo, and re-add it *after* 109 | # your local repository: 110 | remove_repos = ['Solus'] 111 | add_repos = ['Local','Solus'] 112 | 113 | 114 | 115 | ## COPYRIGHT 116 | 117 | * Copyright © 2016 Ikey Doherty, License: CC-BY-SA-3.0 118 | 119 | 120 | ## SEE ALSO 121 | 122 | 123 | `solbuild(1)`, `solbuild.conf(5)` 124 | 125 | https://github.com/toml-lang/toml 126 | 127 | ## NOTES 128 | 129 | Creative Commons Attribution-ShareAlike 3.0 Unported 130 | 131 | * http://creativecommons.org/licenses/by-sa/3.0/ 132 | -------------------------------------------------------------------------------- /src/builder/chroot.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2016-2017 Ikey Doherty 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package builder 18 | 19 | import ( 20 | "fmt" 21 | log "github.com/Sirupsen/logrus" 22 | "github.com/solus-project/libosdev/commands" 23 | "os" 24 | ) 25 | 26 | // Chroot will attempt to spawn a chroot in the overlayfs system 27 | func (p *Package) Chroot(notif PidNotifier, pman *EopkgManager, overlay *Overlay) error { 28 | log.WithFields(log.Fields{ 29 | "profile": overlay.Back.Name, 30 | "version": p.Version, 31 | "package": p.Name, 32 | "type": p.Type, 33 | "release": p.Release, 34 | }).Debug("Beginning chroot") 35 | 36 | var env []string 37 | if p.Type == PackageTypeXML { 38 | env = SaneEnvironment("root", "/root") 39 | } else { 40 | env = SaneEnvironment(BuildUser, BuildUserHome) 41 | } 42 | ChrootEnvironment = env 43 | 44 | if err := p.ActivateRoot(overlay); err != nil { 45 | return err 46 | } 47 | 48 | // Now kill networking 49 | if p.Type == PackageTypeYpkg { 50 | if !p.CanNetwork { 51 | if err := DropNetworking(); err != nil { 52 | return err 53 | } 54 | 55 | // Ensure the overlay can network on localhost only 56 | if err := overlay.ConfigureNetworking(); err != nil { 57 | return err 58 | } 59 | } else { 60 | log.Warning("Package has explicitly requested networking, sandboxing disabled") 61 | } 62 | } 63 | 64 | log.Debug("Spawning login shell") 65 | // Allow bash to work 66 | commands.SetStdin(os.Stdin) 67 | 68 | // Legacy package format requires root, stay as root. 69 | user := BuildUser 70 | if p.Type == PackageTypeXML { 71 | user = "root" 72 | } 73 | 74 | loginCommand := fmt.Sprintf("/bin/su - %s -s %s", user, BuildUserShell) 75 | err := ChrootExecStdin(notif, overlay.MountPoint, loginCommand) 76 | commands.SetStdin(nil) 77 | notif.SetActivePID(0) 78 | return err 79 | } 80 | -------------------------------------------------------------------------------- /src/builder/config.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2016-2017 Ikey Doherty 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package builder 18 | 19 | import ( 20 | "fmt" 21 | "github.com/BurntSushi/toml" 22 | "io/ioutil" 23 | "os" 24 | "path/filepath" 25 | ) 26 | 27 | // Config defines the global defaults for solbuild 28 | type Config struct { 29 | DefaultProfile string `toml:"default_profile"` // Name of the default profile to use 30 | EnableTmpfs bool `toml:"enable_tmpfs"` // Whether to enable tmpfs builds or 31 | TmpfsSize string `toml:"tmpfs_size"` // Bounding size on the tmpfs 32 | } 33 | 34 | var ( 35 | // ConfigPaths is a set of locations for valid solbuild configuration files 36 | ConfigPaths = []string{ 37 | "/etc/solbuild", 38 | "/usr/share/solbuild", 39 | } 40 | 41 | // ConfigSuffix is the suffix a file must have to be glob loaded by solbuild 42 | ConfigSuffix = ".conf" 43 | ) 44 | 45 | // NewConfig will read all the system config files and then the vendor config files 46 | // until it gets somewhere. 47 | func NewConfig() (*Config, error) { 48 | // Set up some sane defaults just in case someone mangles the configs 49 | config := &Config{ 50 | DefaultProfile: "main-x86_64", 51 | EnableTmpfs: false, 52 | TmpfsSize: "", 53 | } 54 | 55 | // Reverse because /etc takes precedence in stateless 56 | for i := len(ConfigPaths) - 1; i >= 0; i-- { 57 | globPat := filepath.Join(ConfigPaths[i], fmt.Sprintf("*%s", ConfigSuffix)) 58 | 59 | configs, _ := filepath.Glob(globPat) 60 | 61 | // Load all globbed configs, using the same Config instance, to keep 62 | // setting the new flags/etc/ 63 | for _, p := range configs { 64 | // Read the config file 65 | fi, err := os.Open(p) 66 | if err != nil { 67 | return nil, err 68 | } 69 | var b []byte 70 | 71 | // We don't defer the close because of the amount of files we could 72 | // potentially glob & open, we don't want to take the piss with open 73 | // file descriptors. 74 | if b, err = ioutil.ReadAll(fi); err != nil { 75 | fi.Close() 76 | return nil, err 77 | } 78 | fi.Close() 79 | 80 | if _, err = toml.Decode(string(b), config); err != nil { 81 | return nil, err 82 | } 83 | } 84 | } 85 | return config, nil 86 | } 87 | -------------------------------------------------------------------------------- /src/builder/copy.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2016-2017 Ikey Doherty 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package builder 18 | 19 | import ( 20 | log "github.com/Sirupsen/logrus" 21 | "github.com/solus-project/libosdev/disk" 22 | "io/ioutil" 23 | "os" 24 | "path/filepath" 25 | ) 26 | 27 | // CopyAll will copy the source asset into the given destdir. 28 | // If the source is a directory, it will be recursively copied 29 | // into the directory destdir. 30 | // 31 | // Note that all directories are created as 00755, as solbuild 32 | // has no interest in the individual folder permissions, just 33 | // the files themselves. 34 | func CopyAll(source, destdir string) error { 35 | // We double stat, get over it. 36 | st, err := os.Stat(source) 37 | // File doesn't exist, move on 38 | if err != nil || st == nil { 39 | return nil 40 | } 41 | 42 | if st.Mode().IsDir() { 43 | var files []os.FileInfo 44 | if files, err = ioutil.ReadDir(source); err != nil { 45 | return err 46 | } 47 | for _, f := range files { 48 | spath := filepath.Join(source, f.Name()) 49 | dpath := filepath.Join(destdir, filepath.Base(source)) 50 | if err := CopyAll(spath, dpath); err != nil { 51 | return err 52 | } 53 | } 54 | } else { 55 | if !PathExists(destdir) { 56 | log.WithFields(log.Fields{ 57 | "dir": destdir, 58 | }).Debug("Creating target directory") 59 | if err = os.MkdirAll(destdir, 00755); err != nil { 60 | log.WithFields(log.Fields{ 61 | "dir": destdir, 62 | "error": err, 63 | }).Error("Failed to create target directory") 64 | return err 65 | } 66 | } 67 | tgt := filepath.Join(destdir, filepath.Base(source)) 68 | log.WithFields(log.Fields{ 69 | "source": source, 70 | "target": tgt, 71 | }).Debug("Copying source asset") 72 | if err = disk.CopyFile(source, tgt); err != nil { 73 | log.WithFields(log.Fields{ 74 | "source": source, 75 | "target": tgt, 76 | "error": err, 77 | }).Error("Failed to copy source asset to target") 78 | return err 79 | } 80 | } 81 | return nil 82 | } 83 | -------------------------------------------------------------------------------- /src/builder/eopkg.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2016-2017 Ikey Doherty 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package builder 18 | 19 | import ( 20 | "fmt" 21 | log "github.com/Sirupsen/logrus" 22 | "github.com/solus-project/libosdev/commands" 23 | "github.com/solus-project/libosdev/disk" 24 | "io/ioutil" 25 | "os" 26 | "path/filepath" 27 | "strings" 28 | ) 29 | 30 | // eopkgCommand utility wraps all eopkg calls to autodisable colours 31 | // where appropriate, as eopkg largely ignores the console type. 32 | func eopkgCommand(c string) string { 33 | if !DisableColors { 34 | return c 35 | } 36 | return fmt.Sprintf("%s -N", c) 37 | } 38 | 39 | // An EopkgRepo is a simplistic representation of a repo found in any given 40 | // chroot. 41 | type EopkgRepo struct { 42 | ID string 43 | URI string 44 | } 45 | 46 | // EopkgManager is our own very shorted version of libosdev EopkgManager, to 47 | // enable very very simple operations 48 | type EopkgManager struct { 49 | dbusActive bool 50 | root string 51 | cacheSource string 52 | cacheTarget string 53 | dbusPid string 54 | 55 | notif PidNotifier 56 | } 57 | 58 | // NewEopkgManager will return a new eopkg manager 59 | func NewEopkgManager(notif PidNotifier, root string) *EopkgManager { 60 | return &EopkgManager{ 61 | dbusActive: false, 62 | root: root, 63 | cacheSource: PackageCacheDirectory, 64 | cacheTarget: filepath.Join(root, "var/cache/eopkg/packages"), 65 | dbusPid: filepath.Join(root, "var/run/dbus/pid"), 66 | notif: notif, 67 | } 68 | } 69 | 70 | // CopyAssets will copy any required host-side assets into the system. This 71 | // function has to be reusable simply because performing an eopkg upgrade 72 | // or installing deps, prior to building, could clobber the files. 73 | func (e *EopkgManager) CopyAssets() error { 74 | requiredAssets := map[string]string{ 75 | "/etc/resolv.conf": filepath.Join(e.root, "etc/resolv.conf"), 76 | "/etc/eopkg/eopkg.conf": filepath.Join(e.root, "etc/eopkg/eopkg.conf"), 77 | } 78 | 79 | for key, value := range requiredAssets { 80 | if !PathExists(key) { 81 | continue 82 | } 83 | dirName := filepath.Dir(value) 84 | if !PathExists(dirName) { 85 | log.WithFields(log.Fields{ 86 | "dir": dirName, 87 | }).Debug("Creating required directory") 88 | if err := os.MkdirAll(dirName, 00755); err != nil { 89 | log.WithFields(log.Fields{ 90 | "dir": dirName, 91 | "error": err, 92 | }).Error("Failed to create required asset directory") 93 | return err 94 | } 95 | } 96 | log.WithFields(log.Fields{ 97 | "file": key, 98 | }).Debug("Copying host asset") 99 | if err := disk.CopyFile(key, value); err != nil { 100 | log.WithFields(log.Fields{ 101 | "file": key, 102 | "error": err, 103 | }).Error("Failed to copy host asset") 104 | return err 105 | } 106 | } 107 | return nil 108 | } 109 | 110 | // Init will do some basic preparation of the chroot 111 | func (e *EopkgManager) Init() error { 112 | // Ensure dbus pid is gone 113 | if PathExists(e.dbusPid) { 114 | if err := os.Remove(e.dbusPid); err != nil { 115 | return err 116 | } 117 | } 118 | 119 | if err := e.CopyAssets(); err != nil { 120 | return err 121 | } 122 | 123 | // Ensure system wide cache exists 124 | if !PathExists(e.cacheSource) { 125 | log.WithFields(log.Fields{ 126 | "dir": e.cacheSource, 127 | }).Debug("Creating system-wide package cache") 128 | if err := os.MkdirAll(e.cacheSource, 00755); err != nil { 129 | log.WithFields(log.Fields{ 130 | "dir": e.cacheSource, 131 | "error": err, 132 | }).Error("Failed to create package cache") 133 | return err 134 | } 135 | } 136 | 137 | if err := os.MkdirAll(e.cacheTarget, 00755); err != nil { 138 | return err 139 | } 140 | return disk.GetMountManager().BindMount(e.cacheSource, e.cacheTarget) 141 | } 142 | 143 | // StartDBUS will bring up dbus within the chroot 144 | func (e *EopkgManager) StartDBUS() error { 145 | if e.dbusActive { 146 | return nil 147 | } 148 | dbusDir := filepath.Join(e.root, "run", "dbus") 149 | if err := os.MkdirAll(dbusDir, 00755); err != nil { 150 | return err 151 | } 152 | if err := ChrootExec(e.notif, e.root, "dbus-uuidgen --ensure"); err != nil { 153 | return err 154 | } 155 | e.notif.SetActivePID(0) 156 | if err := ChrootExec(e.notif, e.root, "dbus-daemon --system"); err != nil { 157 | return err 158 | } 159 | e.notif.SetActivePID(0) 160 | e.dbusActive = true 161 | return nil 162 | } 163 | 164 | // StopDBUS will tear down dbus 165 | func (e *EopkgManager) StopDBUS() error { 166 | // No sense killing dbus twice 167 | if !e.dbusActive { 168 | return nil 169 | } 170 | var b []byte 171 | var err error 172 | var f *os.File 173 | 174 | if f, err = os.Open(e.dbusPid); err != nil { 175 | return err 176 | } 177 | defer func() { 178 | f.Close() 179 | os.Remove(e.dbusPid) 180 | e.dbusActive = false 181 | }() 182 | 183 | if b, err = ioutil.ReadAll(f); err != nil { 184 | return err 185 | } 186 | 187 | pid := strings.Split(string(b), "\n")[0] 188 | return commands.ExecStdoutArgs("kill", []string{"-9", pid}) 189 | } 190 | 191 | // Cleanup will take care of any work we've already done before 192 | func (e *EopkgManager) Cleanup() { 193 | e.StopDBUS() 194 | disk.GetMountManager().Unmount(e.cacheTarget) 195 | } 196 | 197 | // Upgrade will perform an eopkg upgrade inside the chroot 198 | func (e *EopkgManager) Upgrade() error { 199 | // Certain requirements may not be in system.base, but are required for 200 | // proper containerized functionality. 201 | newReqs := []string{ 202 | "iproute2", 203 | } 204 | if err := ChrootExec(e.notif, e.root, eopkgCommand("eopkg upgrade -y")); err != nil { 205 | return err 206 | } 207 | e.notif.SetActivePID(0) 208 | err := ChrootExec(e.notif, e.root, eopkgCommand(fmt.Sprintf("eopkg install -y %s", strings.Join(newReqs, " ")))) 209 | return err 210 | } 211 | 212 | // InstallComponent will install the named component inside the chroot 213 | func (e *EopkgManager) InstallComponent(comp string) error { 214 | err := ChrootExec(e.notif, e.root, eopkgCommand(fmt.Sprintf("eopkg install -c %v -y", comp))) 215 | e.notif.SetActivePID(0) 216 | return err 217 | } 218 | 219 | // EnsureEopkgLayout will enforce changes to the filesystem to make sure that 220 | // it works as expected. 221 | func EnsureEopkgLayout(root string) error { 222 | // Ensures we don't end up with /var/lock vs /run/lock nonsense 223 | reqDirs := []string{ 224 | "run/lock", 225 | "var", 226 | // Enables our bind mounting for caching 227 | "var/cache/eopkg/packages", 228 | } 229 | 230 | // Now we must nuke /run if it exists inside the chroot! 231 | runPath := filepath.Join(root, "run") 232 | if PathExists(runPath) { 233 | if err := os.RemoveAll(runPath); err != nil { 234 | log.WithFields(log.Fields{ 235 | "error": err, 236 | }).Error("Failed to clean stale /run") 237 | return err 238 | } 239 | } 240 | 241 | if err := os.MkdirAll(runPath, 00755); err != nil { 242 | log.WithFields(log.Fields{ 243 | "error": err, 244 | }).Error("Failed to clean create /run") 245 | return err 246 | } 247 | 248 | // Construct the required directories in the tree 249 | for _, dir := range reqDirs { 250 | dirPath := filepath.Join(root, dir) 251 | if err := os.MkdirAll(dirPath, 00755); err != nil { 252 | return err 253 | } 254 | } 255 | 256 | lockTgt := filepath.Join(root, "var", "lock") 257 | if !PathExists(lockTgt) { 258 | if err := os.Symlink("../run/lock", lockTgt); err != nil { 259 | return err 260 | } 261 | } 262 | runTgt := filepath.Join(root, "var", "run") 263 | if !PathExists(runTgt) { 264 | if err := os.Symlink("../run", filepath.Join(root, "var", "run")); err != nil { 265 | return err 266 | } 267 | } 268 | 269 | return nil 270 | } 271 | 272 | // Read the given plaintext URI file to find the target 273 | func readURIFile(path string) (string, error) { 274 | fi, err := os.Open(path) 275 | if err != nil { 276 | return "", err 277 | } 278 | defer fi.Close() 279 | contents, err := ioutil.ReadAll(fi) 280 | if err != nil { 281 | return "", err 282 | } 283 | return string(contents), nil 284 | } 285 | 286 | // GetRepos will attempt to discover all the repos on the target filesystem 287 | func (e *EopkgManager) GetRepos() ([]*EopkgRepo, error) { 288 | globPat := filepath.Join(e.root, "var", "lib", "eopkg", "index", "*", "uri") 289 | var repoFiles []string 290 | 291 | log.Debug("Discovering repos in rootfs") 292 | 293 | repoFiles, _ = filepath.Glob(globPat) 294 | // No repos 295 | if len(repoFiles) < 1 { 296 | return nil, nil 297 | } 298 | 299 | var repos []*EopkgRepo 300 | 301 | for _, repo := range repoFiles { 302 | uri, err := readURIFile(repo) 303 | if err != nil { 304 | log.WithFields(log.Fields{ 305 | "error": err, 306 | "path": repo, 307 | }).Error("Unable to read repository file") 308 | return nil, err 309 | } 310 | repoName := filepath.Base(filepath.Dir(repo)) 311 | repos = append(repos, &EopkgRepo{ 312 | ID: repoName, 313 | URI: uri, 314 | }) 315 | } 316 | return repos, nil 317 | } 318 | 319 | // AddRepo will attempt to add a repo to the filesystem 320 | func (e *EopkgManager) AddRepo(id, source string) error { 321 | e.notif.SetActivePID(0) 322 | return ChrootExec(e.notif, e.root, eopkgCommand(fmt.Sprintf("eopkg add-repo '%s' '%s'", id, source))) 323 | } 324 | 325 | // RemoveRepo will attempt to remove a named repo from the filesystem 326 | func (e *EopkgManager) RemoveRepo(id string) error { 327 | e.notif.SetActivePID(0) 328 | return ChrootExec(e.notif, e.root, eopkgCommand(fmt.Sprintf("eopkg remove-repo '%s'", id))) 329 | } 330 | -------------------------------------------------------------------------------- /src/builder/history.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2016-2017 Ikey Doherty 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package builder 18 | 19 | import ( 20 | "encoding/xml" 21 | "errors" 22 | "fmt" 23 | "github.com/libgit2/git2go" 24 | "os" 25 | "path/filepath" 26 | "regexp" 27 | "sort" 28 | "time" 29 | ) 30 | 31 | const ( 32 | // MaxChangelogEntries is the absolute maximum number of entries we'll 33 | // parse and provide changelog entries for. 34 | MaxChangelogEntries = 10 35 | 36 | // UpdateDateFormat is the time format we emit in the history.xml, i.e. 37 | // 2016-09-24 38 | UpdateDateFormat = "2006-01-02" 39 | ) 40 | 41 | var ( 42 | // CveRegex is used to identify security updates which mention a specific 43 | // CVE ID. 44 | CveRegex *regexp.Regexp 45 | ) 46 | 47 | func init() { 48 | CveRegex = regexp.MustCompile(`(CVE\-[0-9]+\-[0-9]+)`) 49 | } 50 | 51 | // PackageHistory is an automatic changelog generated from the changes to 52 | // the package.yml file during the history of the package. 53 | // 54 | // Through this system, we provide a `history.xml` file to `ypkg-build` 55 | // inside the container, which allows it to export the changelog back to 56 | // the user. 57 | // 58 | // This provides a much more natural system than having dedicated changelog 59 | // files in package gits, as it reduces any and all duplication. 60 | // We also have the opportunity to parse natural elements from the git history 61 | // to make determinations as to the update *type*, such as a security update, 62 | // or an update that requires a reboot to the users system. 63 | // 64 | // Currently we're only scoping for security update notification, though 65 | // more features will come in time. 66 | type PackageHistory struct { 67 | Updates []*PackageUpdate 68 | 69 | pkgfile string // Path of the package 70 | } 71 | 72 | // A PackageUpdate is a point in history in the git changes, which is parsed 73 | // from a git.Commit 74 | type PackageUpdate struct { 75 | Tag string // The associated git tag 76 | Author string // The author name of the change 77 | AuthorEmail string // The author email of the change 78 | Body string // The associated message of the commit 79 | Time time.Time // When the update took place 80 | ObjectID string // OID stored in string form 81 | Package *Package // Associated parsed package 82 | IsSecurity bool // Whether this is a security update 83 | } 84 | 85 | // NewPackageUpdate will attempt to parse the given commit and provide a usable 86 | // entry for the PackageHistory 87 | func NewPackageUpdate(tag string, commit *git.Commit, objectID string) *PackageUpdate { 88 | signature := commit.Author() 89 | update := &PackageUpdate{Tag: tag} 90 | 91 | // We duplicate. cgo makes life difficult. 92 | update.Author = signature.Name 93 | update.AuthorEmail = signature.Email 94 | update.Body = commit.Message() 95 | update.Time = signature.When 96 | update.ObjectID = objectID 97 | 98 | // Attempt to identify the update type. Limit to 1 match, we only need to 99 | // know IF there is a CVE fix, not how many. 100 | cves := CveRegex.FindAllString(update.Body, 1) 101 | if len(cves) > 0 { 102 | update.IsSecurity = true 103 | } 104 | 105 | return update 106 | } 107 | 108 | // CatGitBlob will return the contents of the given entry 109 | func CatGitBlob(repo *git.Repository, entry *git.TreeEntry) ([]byte, error) { 110 | obj, err := repo.Lookup(entry.Id) 111 | if err != nil { 112 | return nil, err 113 | } 114 | blob, err := obj.AsBlob() 115 | if err != nil { 116 | return nil, err 117 | } 118 | return blob.Contents(), nil 119 | } 120 | 121 | // GetFileContents will attempt to read the entire object at path from 122 | // the given tag, within that repo. 123 | func GetFileContents(repo *git.Repository, tag, path string) ([]byte, error) { 124 | oid, err := git.NewOid(tag) 125 | if err != nil { 126 | return nil, err 127 | } 128 | commit, err := repo.Lookup(oid) 129 | if err != nil { 130 | return nil, err 131 | } 132 | treeObj, err := commit.Peel(git.ObjectTree) 133 | if err != nil { 134 | return nil, err 135 | } 136 | tree, err := treeObj.AsTree() 137 | if err != nil { 138 | return nil, err 139 | } 140 | entry, err := tree.EntryByPath(path) 141 | if err != nil { 142 | return nil, err 143 | } 144 | 145 | return CatGitBlob(repo, entry) 146 | } 147 | 148 | // NewPackageHistory will attempt to analyze the git history at the given 149 | // repository path, and return a usable instance of PackageHistory for writing 150 | // to the container history.xml file. 151 | // 152 | // The repository path will be taken as the directory name of the pkgfile that 153 | // is given to this function. 154 | func NewPackageHistory(pkgfile string) (*PackageHistory, error) { 155 | // Repodir 156 | path := filepath.Dir(pkgfile) 157 | 158 | repo, err := git.OpenRepository(path) 159 | if err != nil { 160 | return nil, err 161 | } 162 | // Get all the tags 163 | var tags []string 164 | tags, err = repo.Tags.List() 165 | if err != nil { 166 | return nil, err 167 | } 168 | 169 | updates := make(map[string]*PackageUpdate) 170 | 171 | // Iterate all of the tags 172 | err = repo.Tags.Foreach(func(name string, id *git.Oid) error { 173 | if name == "" || id == nil { 174 | return nil 175 | } 176 | 177 | var commit *git.Commit 178 | 179 | obj, err := repo.Lookup(id) 180 | if err != nil { 181 | return err 182 | } 183 | 184 | switch obj.Type() { 185 | // Unannotated tag 186 | case git.ObjectCommit: 187 | commit, err = obj.AsCommit() 188 | if err != nil { 189 | return err 190 | } 191 | tags = append(tags, name) 192 | // Annotated tag with commit target 193 | case git.ObjectTag: 194 | tag, err := obj.AsTag() 195 | if err != nil { 196 | return err 197 | } 198 | commit, err = repo.LookupCommit(tag.TargetId()) 199 | if err != nil { 200 | return err 201 | } 202 | tags = append(tags, name) 203 | default: 204 | return fmt.Errorf("Internal git error, found %s", obj.Type().String()) 205 | } 206 | if commit == nil { 207 | return nil 208 | } 209 | commitObj := NewPackageUpdate(name, commit, id.String()) 210 | updates[name] = commitObj 211 | return nil 212 | }) 213 | // Foreach went bork 214 | if err != nil { 215 | return nil, err 216 | } 217 | // Sort the tags by -refname 218 | sort.Sort(sort.Reverse(sort.StringSlice(tags))) 219 | 220 | ret := &PackageHistory{pkgfile: pkgfile} 221 | ret.scanUpdates(repo, updates, tags) 222 | updates = nil 223 | 224 | if len(ret.Updates) < 1 { 225 | return nil, errors.New("No usable git history found") 226 | } 227 | 228 | // All done! 229 | return ret, nil 230 | } 231 | 232 | // SortUpdatesByRelease is a simple wrapper to allowing sorting history 233 | type SortUpdatesByRelease []*PackageUpdate 234 | 235 | func (a SortUpdatesByRelease) Len() int { 236 | return len(a) 237 | } 238 | 239 | func (a SortUpdatesByRelease) Swap(i, j int) { 240 | a[i], a[j] = a[j], a[i] 241 | } 242 | 243 | func (a SortUpdatesByRelease) Less(i, j int) bool { 244 | return a[i].Package.Release < a[j].Package.Release 245 | } 246 | 247 | // scanUpdates will go back through the collected, "ok" tags, and analyze 248 | // them to be more useful. 249 | func (p *PackageHistory) scanUpdates(repo *git.Repository, updates map[string]*PackageUpdate, tags []string) { 250 | // basename of file 251 | fname := filepath.Base(p.pkgfile) 252 | 253 | var updateSet []*PackageUpdate 254 | // Iterate the commit set in order 255 | for _, tagID := range tags { 256 | update := updates[tagID] 257 | if update == nil { 258 | continue 259 | } 260 | b, err := GetFileContents(repo, update.ObjectID, fname) 261 | if err != nil { 262 | continue 263 | } 264 | 265 | var pkg *Package 266 | // Shouldn't *actually* bail here. Malformed packages do happen 267 | if pkg, err = NewYmlPackageFromBytes(b); err != nil { 268 | continue 269 | } 270 | update.Package = pkg 271 | updateSet = append(updateSet, update) 272 | } 273 | sort.Sort(sort.Reverse(SortUpdatesByRelease(updateSet))) 274 | if len(updateSet) >= MaxChangelogEntries { 275 | p.Updates = updateSet[:MaxChangelogEntries] 276 | } else { 277 | p.Updates = updateSet 278 | } 279 | 280 | } 281 | 282 | // YPKG provides ypkg-gen-history history.xml compatibility 283 | type YPKG struct { 284 | History []*YPKGUpdate `xml:">Update"` 285 | } 286 | 287 | // YPKGUpdate represents an update in the package history 288 | type YPKGUpdate struct { 289 | Release int `xml:"release,attr"` 290 | Type string `xml:"type,attr,omitempty"` 291 | Date string 292 | Version string 293 | Comment struct { 294 | Value string `xml:",cdata"` 295 | } 296 | Name struct { 297 | Value string `xml:",cdata"` 298 | } 299 | Email string 300 | } 301 | 302 | // WriteXML will attempt to dump the update history to an XML file 303 | // in order for ypkg to merge it into the package build. 304 | func (p *PackageHistory) WriteXML(path string) error { 305 | var ypkgUpdates []*YPKGUpdate 306 | 307 | fi, err := os.Create(path) 308 | if err != nil { 309 | return err 310 | } 311 | defer fi.Close() 312 | 313 | for _, update := range p.Updates { 314 | yUpdate := &YPKGUpdate{ 315 | Release: update.Package.Release, 316 | Version: update.Package.Version, 317 | Email: update.AuthorEmail, 318 | Date: update.Time.Format(UpdateDateFormat), 319 | } 320 | yUpdate.Comment.Value = update.Body 321 | yUpdate.Name.Value = update.Author 322 | if update.IsSecurity { 323 | yUpdate.Type = "security" 324 | } 325 | ypkgUpdates = append(ypkgUpdates, yUpdate) 326 | } 327 | 328 | ypkg := &YPKG{History: ypkgUpdates} 329 | bytes, err := xml.MarshalIndent(ypkg, "", " ") 330 | if err != nil { 331 | return err 332 | } 333 | 334 | // Dump it to the file 335 | _, err = fi.WriteString(string(bytes)) 336 | return err 337 | } 338 | 339 | // GetLastVersionTimestamp will return a timestamp appropriate for us within 340 | // reproducible builds. 341 | // 342 | // This is calculated by using the timestamp from the last explicit version 343 | // change, and not from simple bumps. The idea here is to only increment the 344 | // timestamp if we've actually upgraded to a major version, and in general 345 | // attempt to reduce the noise, and thus, produce better delta packages 346 | // between minor package alterations 347 | func (p *PackageHistory) GetLastVersionTimestamp() int64 { 348 | lastVersion := p.Updates[0].Package.Version 349 | lastTime := p.Updates[0].Time 350 | 351 | if len(p.Updates) < 2 { 352 | return lastTime.UTC().Unix() 353 | } 354 | 355 | // Walk history and find the last version change, assigning timestamp 356 | // as appropriate. 357 | for i := 1; i < len(p.Updates); i++ { 358 | newVersion := p.Updates[i].Package.Version 359 | if newVersion != lastVersion { 360 | break 361 | } 362 | lastVersion = p.Updates[i].Package.Version 363 | lastTime = p.Updates[i].Time 364 | } 365 | 366 | return lastTime.UTC().Unix() 367 | } 368 | -------------------------------------------------------------------------------- /src/builder/index.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2016-2017 Ikey Doherty 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package builder 18 | 19 | import ( 20 | "errors" 21 | "fmt" 22 | log "github.com/Sirupsen/logrus" 23 | "github.com/solus-project/libosdev/disk" 24 | "os" 25 | "path/filepath" 26 | ) 27 | 28 | var ( 29 | // ErrCannotContinue is a stock error return 30 | ErrCannotContinue = errors.New("Index cannot continue") 31 | 32 | // IndexBindTarget is where we always mount the repo 33 | IndexBindTarget = "/hostRepo/Index" 34 | ) 35 | 36 | // Index will attempt to index the given directory 37 | func (p *Package) Index(notif PidNotifier, dir string, overlay *Overlay) error { 38 | log.WithFields(log.Fields{ 39 | "profile": overlay.Back.Name, 40 | }).Debug("Beginning indexer") 41 | 42 | mman := disk.GetMountManager() 43 | 44 | ChrootEnvironment = SaneEnvironment("root", "/root") 45 | 46 | // Check the source exists first! 47 | if !PathExists(dir) { 48 | log.WithFields(log.Fields{ 49 | "dir": dir, 50 | }).Error("Directory does not exist") 51 | return ErrCannotContinue 52 | } 53 | 54 | // Indexer will always create new dirs.. 55 | if err := overlay.CleanExisting(); err != nil { 56 | return err 57 | } 58 | 59 | if err := p.ActivateRoot(overlay); err != nil { 60 | return err 61 | } 62 | 63 | // Create the target 64 | target := filepath.Join(overlay.MountPoint, IndexBindTarget[1:]) 65 | if err := os.MkdirAll(target, 00755); err != nil { 66 | log.WithFields(log.Fields{ 67 | "dir": target, 68 | "error": err, 69 | }).Error("Cannot create bind target") 70 | return err 71 | } 72 | 73 | log.WithFields(log.Fields{ 74 | "dir": dir, 75 | }).Debug("Bind mounting directory for indexing") 76 | 77 | if err := mman.BindMount(dir, target); err != nil { 78 | log.WithFields(log.Fields{ 79 | "dir": target, 80 | "error": err, 81 | }).Error("Cannot bind mount directory") 82 | return err 83 | } 84 | 85 | // Ensure it gets cleaned up 86 | overlay.ExtraMounts = append(overlay.ExtraMounts, target) 87 | 88 | log.Debug("Now indexing") 89 | command := fmt.Sprintf("cd %s; %s", IndexBindTarget, eopkgCommand("eopkg index --skip-signing .")) 90 | if err := ChrootExec(notif, overlay.MountPoint, command); err != nil { 91 | log.WithFields(log.Fields{ 92 | "error": err, 93 | "dir": dir, 94 | }).Error("Indexing failed") 95 | return err 96 | } 97 | return nil 98 | } 99 | -------------------------------------------------------------------------------- /src/builder/lockfile.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2016-2017 Ikey Doherty 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package builder 18 | 19 | import ( 20 | "errors" 21 | "fmt" 22 | "os" 23 | "path/filepath" 24 | "sync" 25 | "syscall" 26 | ) 27 | 28 | var ( 29 | // ErrDeadLockFile is returned when an dead lockfile was encountered 30 | ErrDeadLockFile = errors.New("Dead lockfile") 31 | 32 | // ErrOwnedLockFile is returned when the lockfile is already owned by 33 | // another active process. 34 | ErrOwnedLockFile = errors.New("File is locked") 35 | ) 36 | 37 | // A LockFile encapsulates locking functionality 38 | type LockFile struct { 39 | path string // Path of the lockfile 40 | owningPID int // Process ID of the lockfile owner 41 | ourPID int // Our own process ID 42 | conlock *sync.RWMutex // Concurrency lock for library use 43 | fd *os.File // Actual file being locked 44 | owner bool // Whether we're the owner.. 45 | } 46 | 47 | // NewLockFile will return a new lockfile for the given path 48 | func NewLockFile(path string) (*LockFile, error) { 49 | lock := &LockFile{ 50 | path: path, 51 | owningPID: -1, 52 | ourPID: os.Getpid(), 53 | conlock: new(sync.RWMutex), 54 | fd: nil, 55 | owner: false, 56 | } 57 | 58 | // Automatically create the leading directory structure 59 | dir := filepath.Dir(path) 60 | if !PathExists(dir) { 61 | if err := os.MkdirAll(dir, 00755); err != nil { 62 | return nil, err 63 | } 64 | } 65 | 66 | // We can consider setting the permissions to 0600 67 | w, err := os.OpenFile(lock.path, os.O_RDWR|os.O_CREATE, 00644) 68 | if err != nil { 69 | return nil, err 70 | } 71 | // Store the file descriptor 72 | lock.fd = w 73 | 74 | return lock, nil 75 | } 76 | 77 | // GetOwnerPID will return the owner PID, if it exists 78 | func (l *LockFile) GetOwnerPID() int { 79 | return l.owningPID 80 | } 81 | 82 | // GetOwnerProcess will return the executable name if possible 83 | func (l *LockFile) GetOwnerProcess() string { 84 | fp := fmt.Sprintf("/proc/%d/exe", l.owningPID) 85 | str, err := filepath.EvalSymlinks(fp) 86 | if err != nil { 87 | return "unknown process" 88 | } 89 | return str 90 | } 91 | 92 | // Lock will attempt to lock the file, or return an error if this fails 93 | func (l *LockFile) Lock() error { 94 | pid, err := l.readPID() 95 | 96 | // Bail now. 97 | if err != ErrDeadLockFile && err != ErrOwnedLockFile && err != nil { 98 | return err 99 | } 100 | 101 | // Not gonna test our *own* PID 102 | if pid > 0 && pid != l.ourPID { 103 | // Process is still active 104 | // Unix this always works 105 | p, _ := os.FindProcess(pid) 106 | if err2 := p.Signal(syscall.Signal(0)); err2 == nil { 107 | if p.Pid != l.ourPID { 108 | l.owningPID = p.Pid 109 | return ErrOwnedLockFile 110 | } 111 | } 112 | } 113 | 114 | l.conlock.Lock() 115 | 116 | // Finally lock it. 117 | if err := syscall.Flock(int(l.fd.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { 118 | l.conlock.Unlock() 119 | return err 120 | } 121 | 122 | l.owner = true 123 | 124 | l.conlock.Unlock() 125 | 126 | // Write the PID now we have an exclusive lock on it 127 | return l.writePID() 128 | } 129 | 130 | // Unlock will attempt to unlock the file, or return an error if this fails 131 | func (l *LockFile) Unlock() error { 132 | if l.fd == nil || !l.owner { 133 | return errors.New("cannot unlock that which we don't own") 134 | } 135 | 136 | return syscall.Flock(int(l.fd.Fd()), syscall.LOCK_UN) 137 | } 138 | 139 | // readPID is a simple utility to extract the PID from a file 140 | func (l *LockFile) readPID() (int, error) { 141 | l.conlock.RLock() 142 | defer l.conlock.RUnlock() 143 | 144 | fi, err := os.Open(l.path) 145 | // Likely a permission issue. 146 | if err != nil { 147 | return -1, err 148 | } 149 | defer fi.Close() 150 | var pid int 151 | var n int 152 | 153 | // This is ok, we can just nuke it.. 154 | if n, err = fmt.Fscanf(fi, "%d", &pid); err != nil { 155 | return -1, ErrDeadLockFile 156 | } 157 | // This is actually ok. 158 | if n != 1 { 159 | return -1, ErrDeadLockFile 160 | } 161 | return pid, nil 162 | } 163 | 164 | // writePID will store our PID in the lockfile 165 | func (l *LockFile) writePID() error { 166 | if l.fd == nil { 167 | panic(errors.New("cannot write PID for no file")) 168 | } 169 | l.conlock.Lock() 170 | defer l.conlock.Unlock() 171 | if _, err := fmt.Fprintf(l.fd, "%d", l.ourPID); err != nil { 172 | return err 173 | } 174 | return l.fd.Sync() 175 | } 176 | 177 | // Clean will dispose of the lock file and hopefully the lockfile itself 178 | func (l *LockFile) Clean() error { 179 | l.conlock.Lock() 180 | defer l.conlock.Unlock() 181 | if l.fd == nil || !l.owner { 182 | return nil 183 | } 184 | 185 | l.fd.Close() 186 | if l.owner { 187 | return os.Remove(l.path) 188 | } 189 | return nil 190 | } 191 | -------------------------------------------------------------------------------- /src/builder/main.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2016-2017 Ikey Doherty 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | // Package builder provides all the solbuild specific functionality 18 | package builder 19 | 20 | import ( 21 | "fmt" 22 | "os" 23 | "path/filepath" 24 | ) 25 | 26 | // DisableColors controls whether or not to use colours in the display. 27 | // Spelled this way so people don't get confused :P 28 | var DisableColors bool 29 | 30 | const ( 31 | // ImagesDir is where we keep the rootfs images for build profiles 32 | ImagesDir = "/var/lib/solbuild/images" 33 | 34 | // ImageSuffix is the common suffix for all solbuild images 35 | ImageSuffix = ".img" 36 | 37 | // ImageCompressedSuffix is the common suffix for a fetched evobuild image 38 | ImageCompressedSuffix = ".img.xz" 39 | 40 | // ImageBaseURI is the storage area for base images 41 | ImageBaseURI = "https://solus-project.com/image_root" 42 | 43 | // ImageRootsDir is where updates are performed on base images 44 | ImageRootsDir = "/var/lib/solbuild/roots" 45 | ) 46 | 47 | const ( 48 | // PackageCacheDirectory is where we share packages between all builders 49 | PackageCacheDirectory = "/var/lib/solbuild/packages" 50 | 51 | // CcacheDirectory is the system wide ccache directory 52 | CcacheDirectory = "/var/lib/solbuild/ccache/ypkg" 53 | 54 | // LegacyCcacheDirectory is the root owned ccache directory for pspec.xml 55 | LegacyCcacheDirectory = "/var/lib/solbuild/ccache/legacy" 56 | ) 57 | 58 | const ( 59 | // BuildUser is the user that builds will run as inside the chroot 60 | BuildUser = "build" 61 | 62 | // BuildUserID is the build user's numerical ID 63 | BuildUserID = 1000 64 | 65 | // BuildUserGID is the group to use 66 | BuildUserGID = 1000 67 | 68 | // BuildUserHome is the build user's home directory 69 | BuildUserHome = "/home/build" 70 | 71 | // BuildUserGecos is the build user's description 72 | BuildUserGecos = "solbuild user" 73 | 74 | // BuildUserShell is the system shell for the build user 75 | BuildUserShell = "/bin/bash" 76 | ) 77 | 78 | var ( 79 | // ValidImages is a set of known, Solus-published, base profiles 80 | ValidImages = []string{ 81 | "main-x86_64", 82 | "unstable-x86_64", 83 | } 84 | ) 85 | 86 | // PathExists is a helper function to determine the existence of a file path 87 | func PathExists(path string) bool { 88 | if st, err := os.Stat(path); err == nil && st != nil { 89 | return true 90 | } 91 | return false 92 | } 93 | 94 | // IsValidImage will check if the specified profile is a valid one. 95 | func IsValidImage(profile string) bool { 96 | for _, p := range ValidImages { 97 | if p == profile { 98 | return true 99 | } 100 | } 101 | return false 102 | } 103 | 104 | // EmitImageError emits the stock response to requesting an invalid image 105 | func EmitImageError(image string) { 106 | fmt.Fprintf(os.Stderr, "Error: '%v' is not a known image\n", image) 107 | fmt.Fprintf(os.Stderr, "Valid images include:\n\n") 108 | for _, p := range ValidImages { 109 | fmt.Fprintf(os.Stderr, " * %v\n", p) 110 | } 111 | } 112 | 113 | // EmitProfileError emits a stock response for an invalid profile 114 | func EmitProfileError(p string) { 115 | fmt.Fprintf(os.Stderr, "Error: '%v' is not a known profile\n", p) 116 | fmt.Fprintf(os.Stderr, "Valid profiles include:\n\n") 117 | 118 | profiles, err := GetAllProfiles() 119 | if err != nil { 120 | fmt.Fprintf(os.Stderr, "Error loading profiles: %v\n", err) 121 | return 122 | } 123 | if len(profiles) < 1 { 124 | fmt.Fprintf(os.Stderr, "Fatal: No profiles installed. Reinstall solbuild\n") 125 | return 126 | } 127 | 128 | for key := range profiles { 129 | fmt.Fprintf(os.Stderr, " * %v\n", key) 130 | } 131 | } 132 | 133 | // A BackingImage is the core of any given profile 134 | type BackingImage struct { 135 | Name string // Name of the profile 136 | ImagePath string // Absolute path to the .img file 137 | ImagePathXZ string // Absolute path to the .img.xz file 138 | ImageURI string // URI of the image origin 139 | RootDir string // Where to mount the backing image for updates 140 | LockPath string // Our lock path for update operations 141 | } 142 | 143 | // IsInstalled will determine whether the given backing image has been installed 144 | // to the global image directory or not. 145 | func (b *BackingImage) IsInstalled() bool { 146 | return PathExists(b.ImagePath) 147 | } 148 | 149 | // IsFetched will determine whether or not the XZ image itself has been fetched 150 | func (b *BackingImage) IsFetched() bool { 151 | return PathExists(b.ImagePathXZ) 152 | } 153 | 154 | // NewBackingImage will return a correctly configured backing image for 155 | // usage. 156 | func NewBackingImage(name string) *BackingImage { 157 | return &BackingImage{ 158 | Name: name, 159 | ImagePath: filepath.Join(ImagesDir, name+ImageSuffix), 160 | ImagePathXZ: filepath.Join(ImagesDir, name+ImageCompressedSuffix), 161 | ImageURI: fmt.Sprintf("%s/%s%s", ImageBaseURI, name, ImageCompressedSuffix), 162 | LockPath: filepath.Join(ImagesDir, name+".lock"), 163 | RootDir: filepath.Join(ImageRootsDir, name), 164 | } 165 | } 166 | -------------------------------------------------------------------------------- /src/builder/mmap.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2017 Solus Project 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package builder 18 | 19 | import ( 20 | "os" 21 | "syscall" 22 | ) 23 | 24 | // An MmapFile is used to easily wrap the syscall mmap() functions to be readily 25 | // usable from golang. 26 | // This helps heaps (pun intended) when it comes to computing the hash sum for 27 | // very large files, such as the index and eopkg files, in a zero copy fashion. 28 | type MmapFile struct { 29 | f *os.File 30 | Data []byte 31 | len int64 32 | m bool 33 | } 34 | 35 | // MapFile will attempt to mmap() the input file 36 | func MapFile(path string) (*MmapFile, error) { 37 | var err error 38 | ret := &MmapFile{} 39 | ret.f, err = os.Open(path) 40 | if err != nil { 41 | return nil, err 42 | } 43 | st, err := ret.f.Stat() 44 | if err != nil { 45 | ret.f.Close() 46 | return nil, err 47 | } 48 | ret.len = st.Size() 49 | ret.Data, err = syscall.Mmap(int(ret.f.Fd()), 0, int(ret.len), syscall.PROT_READ, syscall.MAP_PRIVATE) 50 | if err != nil { 51 | ret.f.Close() 52 | return nil, err 53 | } 54 | ret.m = true 55 | return ret, nil 56 | } 57 | 58 | // Close will close the previously mmapped filed 59 | func (m *MmapFile) Close() error { 60 | var err error 61 | if m.f == nil { 62 | return nil 63 | } 64 | if m.m { 65 | err = syscall.Munmap(m.Data) 66 | m.m = false 67 | } 68 | m.f.Close() 69 | m.f = nil 70 | return err 71 | } 72 | -------------------------------------------------------------------------------- /src/builder/namespaces.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2016-2017 Ikey Doherty 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package builder 18 | 19 | import ( 20 | log "github.com/Sirupsen/logrus" 21 | "syscall" 22 | ) 23 | 24 | // ConfigureNamespace will unshare() context, entering a new namespace 25 | func ConfigureNamespace() error { 26 | log.Debug("Configuring container namespace") 27 | if err := syscall.Unshare(syscall.CLONE_NEWNS | syscall.CLONE_NEWIPC); err != nil { 28 | log.WithFields(log.Fields{ 29 | "error": err, 30 | }).Error("Failed to configure namespace") 31 | return err 32 | } 33 | return nil 34 | } 35 | 36 | // DropNetworking will unshare() the context networking capabilities 37 | func DropNetworking() error { 38 | log.Debug("Dropping container networking") 39 | if err := syscall.Unshare(syscall.CLONE_NEWNET | syscall.CLONE_NEWUTS); err != nil { 40 | log.WithFields(log.Fields{ 41 | "error": err, 42 | }).Error("Failed to drop networking capabilities") 43 | return err 44 | } 45 | return nil 46 | } 47 | -------------------------------------------------------------------------------- /src/builder/pkg.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2016-2017 Ikey Doherty 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package builder 18 | 19 | import ( 20 | "builder/source" 21 | "encoding/xml" 22 | "errors" 23 | "fmt" 24 | "github.com/go-yaml/yaml" 25 | "io/ioutil" 26 | "os" 27 | "strings" 28 | ) 29 | 30 | // PackageType is simply the type of package we're building, i.e. xml / pspec 31 | type PackageType string 32 | 33 | const ( 34 | // PackageTypeXML is the legacy package format, to be removed with sol introduction. 35 | PackageTypeXML PackageType = "legacy" 36 | 37 | // PackageTypeYpkg is the native build format of Solus, the package.yml format 38 | PackageTypeYpkg PackageType = "ypkg" 39 | 40 | // PackageTypeIndex is a faux type to enable indexing 41 | PackageTypeIndex PackageType = "index" 42 | ) 43 | 44 | var ( 45 | // IndexPackage is used by the index command to make use of the overlayfs 46 | // system. 47 | IndexPackage = Package{ 48 | Name: "index", 49 | Version: "1.4.2", 50 | Type: PackageTypeIndex, 51 | Release: 1, 52 | Path: "", 53 | } 54 | ) 55 | 56 | // Package is the main item we deal with, avoiding the internals 57 | type Package struct { 58 | Name string // Name of the package 59 | Version string // Version of this package 60 | Release int // Solus upgrades are based entirely on relno 61 | Type PackageType // ypkg or pspec.xml legacy 62 | Path string // Path to the build spec 63 | Sources []source.Source // Each package has 0 or more sources that we fetch 64 | CanNetwork bool // Only applicable to ypkg builds 65 | } 66 | 67 | // YmlPackage is a parsed ypkg build file 68 | type YmlPackage struct { 69 | Name string 70 | Version string 71 | Release int 72 | Networking bool // If set to false (default) we disable networking in the build 73 | Source []map[string]string 74 | } 75 | 76 | // XMLUpdate represents an update in the package history 77 | type XMLUpdate struct { 78 | Release int `xml:"release,attr"` 79 | Date string 80 | Version string 81 | Comment string 82 | Name string 83 | Email string 84 | } 85 | 86 | // XMLArchive is an line in Source section 87 | type XMLArchive struct { 88 | Type string `xml:"type,attr"` 89 | SHA1Sum string `xml:"sha1sum,attr"` 90 | URI string `xml:",chardata"` 91 | } 92 | 93 | // XMLSource is the actual source info for each pspec.xml 94 | type XMLSource struct { 95 | Homepage string 96 | Name string 97 | Archive []XMLArchive 98 | } 99 | 100 | // XMLPackage contains all of the pspec.xml metadata 101 | type XMLPackage struct { 102 | Name string 103 | Source XMLSource 104 | History []XMLUpdate `xml:"History>Update"` 105 | } 106 | 107 | // NewPackage will attempt to parse the given path, and return a new Package 108 | // instance if this succeeds. 109 | func NewPackage(path string) (*Package, error) { 110 | if strings.HasSuffix(path, ".xml") { 111 | return NewXMLPackage(path) 112 | } 113 | return NewYmlPackage(path) 114 | } 115 | 116 | // NewXMLPackage will attempt to parse the pspec.xml file @ path 117 | func NewXMLPackage(path string) (*Package, error) { 118 | var by []byte 119 | var err error 120 | var fi *os.File 121 | 122 | fi, err = os.Open(path) 123 | if err != nil { 124 | return nil, err 125 | } 126 | defer fi.Close() 127 | 128 | by, err = ioutil.ReadAll(fi) 129 | if err != nil { 130 | return nil, err 131 | } 132 | xpkg := &XMLPackage{} 133 | if err = xml.Unmarshal(by, xpkg); err != nil { 134 | return nil, err 135 | } 136 | if len(xpkg.History) < 1 { 137 | return nil, errors.New("xml: Malformed pspec file") 138 | } 139 | 140 | upd := xpkg.History[0] 141 | ret := &Package{ 142 | Name: strings.TrimSpace(xpkg.Source.Name), 143 | Version: strings.TrimSpace(upd.Version), 144 | Release: upd.Release, 145 | Type: PackageTypeXML, 146 | Path: path, 147 | CanNetwork: true, 148 | } 149 | 150 | for _, archive := range xpkg.Source.Archive { 151 | source, err := source.New(archive.URI, archive.SHA1Sum, true) 152 | if err != nil { 153 | return nil, err 154 | } 155 | ret.Sources = append(ret.Sources, source) 156 | } 157 | 158 | if ret.Name == "" { 159 | return nil, errors.New("xml: Missing name in package") 160 | } 161 | if ret.Version == "" { 162 | return nil, errors.New("xml: Missing version in package") 163 | } 164 | if ret.Release < 0 { 165 | return nil, fmt.Errorf("xml: Invalid release in package: %d", ret.Release) 166 | } 167 | return ret, nil 168 | } 169 | 170 | // NewYmlPackage will attempt to parse the ypkg package.yml file @ path 171 | func NewYmlPackage(path string) (*Package, error) { 172 | var by []byte 173 | var err error 174 | var fi *os.File 175 | 176 | fi, err = os.Open(path) 177 | if err != nil { 178 | return nil, err 179 | } 180 | defer fi.Close() 181 | 182 | by, err = ioutil.ReadAll(fi) 183 | if err != nil { 184 | return nil, err 185 | } 186 | ret, err := NewYmlPackageFromBytes(by) 187 | if err != nil { 188 | return nil, err 189 | } 190 | ret.Path = path 191 | return ret, nil 192 | } 193 | 194 | // NewYmlPackageFromBytes will attempt to parse the ypkg package.yml in memory 195 | func NewYmlPackageFromBytes(by []byte) (*Package, error) { 196 | var err error 197 | 198 | ypkg := &YmlPackage{Networking: false} 199 | if err = yaml.Unmarshal(by, ypkg); err != nil { 200 | return nil, err 201 | } 202 | 203 | ret := &Package{ 204 | Name: strings.TrimSpace(ypkg.Name), 205 | Version: strings.TrimSpace(ypkg.Version), 206 | Release: ypkg.Release, 207 | Type: PackageTypeYpkg, 208 | CanNetwork: ypkg.Networking, 209 | } 210 | 211 | for _, row := range ypkg.Source { 212 | for key, value := range row { 213 | source, err := source.New(key, value, false) 214 | if err != nil { 215 | return nil, err 216 | } 217 | ret.Sources = append(ret.Sources, source) 218 | } 219 | } 220 | 221 | if ret.Name == "" { 222 | return nil, errors.New("ypkg: Missing name in package") 223 | } 224 | if ret.Version == "" { 225 | return nil, errors.New("ypkg: Missing version in package") 226 | } 227 | if ret.Release < 0 { 228 | return nil, fmt.Errorf("ypkg: Invalid release in package: %d", ret.Release) 229 | } 230 | return ret, nil 231 | } 232 | -------------------------------------------------------------------------------- /src/builder/profile.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2016-2017 Ikey Doherty 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package builder 18 | 19 | import ( 20 | "fmt" 21 | "github.com/BurntSushi/toml" 22 | "io/ioutil" 23 | "os" 24 | "path/filepath" 25 | "strings" 26 | ) 27 | 28 | // A Repo is a definition of a repository to add to the eopkg root during 29 | // the build process. 30 | type Repo struct { 31 | Name string `toml:"-"` // Name of the repo, set by implementation not yoml 32 | URI string `toml:"uri"` // URI of the repository 33 | Local bool `toml:"local"` // Local repository for bindmounting 34 | AutoIndex bool `toml:"autoindex"` // Enable automatic indexing of the repo 35 | } 36 | 37 | // A Profile is a configuration defining what backing image to use, what repos 38 | // to add, etc. 39 | type Profile struct { 40 | Name string `toml:"-"` // Name of this profile, set by file name not toml 41 | Image string `toml:"image"` // The backing image for this profile 42 | RemoveRepos []string `toml:"remove_repos"` // A set of repos to remove. ["*"] is valid here. 43 | Repos map[string]*Repo `toml:"repo"` // Allow defining custom repos 44 | AddRepos []string `toml:"add_repos"` // Allow locking to a single set of repos 45 | } 46 | 47 | var ( 48 | // ProfileSuffix is the fixed extension for solbuild profile files 49 | ProfileSuffix = ".profile" 50 | ) 51 | 52 | // NewProfile will attempt to load the named profile from the system paths 53 | func NewProfile(name string) (*Profile, error) { 54 | for _, p := range ConfigPaths { 55 | fp := filepath.Join(p, fmt.Sprintf("%s%s", name, ProfileSuffix)) 56 | if !PathExists(fp) { 57 | continue 58 | } 59 | return NewProfileFromPath(fp) 60 | } 61 | return nil, ErrInvalidProfile 62 | } 63 | 64 | // GetAllProfiles will locate all available profiles for solbuild 65 | func GetAllProfiles() (map[string]*Profile, error) { 66 | ret := make(map[string]*Profile) 67 | 68 | for _, p := range ConfigPaths { 69 | gl := filepath.Join(p, "*.profile") 70 | 71 | profiles, _ := filepath.Glob(gl) 72 | 73 | for _, o := range profiles { 74 | if profile, err := NewProfileFromPath(o); err == nil { 75 | ret[profile.Name] = profile 76 | } else { 77 | return nil, err 78 | } 79 | } 80 | } 81 | return ret, nil 82 | } 83 | 84 | // NewProfileFromPath will attempt to load a profile from the given file name 85 | func NewProfileFromPath(path string) (*Profile, error) { 86 | basename := filepath.Base(path) 87 | if !strings.HasSuffix(basename, ProfileSuffix) { 88 | return nil, fmt.Errorf("Not a .profile file: %v", path) 89 | } 90 | 91 | fi, err := os.Open(path) 92 | if err != nil { 93 | return nil, err 94 | } 95 | defer fi.Close() 96 | 97 | profileName := basename[:len(basename)-len(ProfileSuffix)] 98 | 99 | var b []byte 100 | profile := &Profile{Name: profileName} 101 | 102 | // Read the config file 103 | if b, err = ioutil.ReadAll(fi); err != nil { 104 | return nil, err 105 | } 106 | 107 | if _, err = toml.Decode(string(b), profile); err != nil { 108 | return nil, err 109 | } 110 | 111 | // Ensure all repos have a valid name 112 | for name, repo := range profile.Repos { 113 | repo.Name = name 114 | } 115 | 116 | // Ignore a wildcard add 117 | if len(profile.AddRepos) == 1 && profile.AddRepos[0] == "*" { 118 | return profile, nil 119 | } 120 | 121 | // Check all repo names are valid 122 | for _, r := range profile.AddRepos { 123 | if _, ok := profile.Repos[r]; !ok { 124 | return nil, fmt.Errorf("Cannot enable unknown repo %v", r) 125 | } 126 | } 127 | 128 | return profile, nil 129 | } 130 | -------------------------------------------------------------------------------- /src/builder/profile_test.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2016-2017 Ikey Doherty 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package builder 18 | 19 | import ( 20 | "testing" 21 | ) 22 | 23 | const ( 24 | ProfileTestFile = "testdata/unstable.profile" 25 | ) 26 | 27 | func TestLoadProfile(t *testing.T) { 28 | if _, err := NewProfileFromPath("@'werlq;krqr8u3283"); err == nil { 29 | t.Fatal("Loaded a file that doesn't exist!") 30 | } 31 | 32 | profile, err := NewProfileFromPath(ProfileTestFile) 33 | if err != nil { 34 | t.Fatalf("Failed to load configuration from valid path: %v", err) 35 | } 36 | if profile == nil { 37 | t.Fatal("No error but nil profile") 38 | } 39 | if profile.Image != "unstable-x86_64" { 40 | t.Fatalf("Wrong image in profile: %v", profile.Image) 41 | } 42 | if repo, ok := profile.Repos["Solus"]; ok { 43 | if repo.URI != "https://packages.solus-project.com/unstable/eopkg-index.xml.xz" { 44 | t.Fatalf("Wrong Solus URI: %v", repo.URI) 45 | } 46 | } else { 47 | t.Fatal("Missing Solus repo") 48 | } 49 | if _, ok := profile.Repos["Bob"]; ok { 50 | t.Fatal("Should not have a repo here!") 51 | } 52 | if len(profile.RemoveRepos) != 0 { 53 | t.Fatalf("Invalid number of remove repos: %d", len(profile.RemoveRepos)) 54 | } 55 | if len(profile.AddRepos) != 1 { 56 | t.Fatalf("Invalid number of add repos: %d", len(profile.AddRepos)) 57 | } 58 | if len(profile.Repos) != 3 { 59 | t.Fatalf("Invalid number of repos: %d", len(profile.Repos)) 60 | } 61 | if profile.AddRepos[0] != "Solus" { 62 | t.Fatalf("Invalid AddRepos: %s", profile.AddRepos[0]) 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/builder/repos.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2016-2017 Ikey Doherty 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package builder 18 | 19 | import ( 20 | "fmt" 21 | log "github.com/Sirupsen/logrus" 22 | "github.com/solus-project/libosdev/disk" 23 | "os" 24 | "path/filepath" 25 | ) 26 | 27 | const ( 28 | // BindRepoDir is where we make repos available from the host side 29 | BindRepoDir = "/hostRepos" 30 | ) 31 | 32 | // addLocalRepo will try to add the repo and bind mount it into the target 33 | func (p *Package) addLocalRepo(notif PidNotifier, o *Overlay, pkgManager *EopkgManager, repo *Repo) error { 34 | // Ensure the source exists too. Sorta helpful like that. 35 | if !PathExists(repo.URI) { 36 | return fmt.Errorf("Local repo does not exist") 37 | } 38 | 39 | mman := disk.GetMountManager() 40 | 41 | // Ensure the target mountpoint actually exists ... 42 | tgt := filepath.Join(o.MountPoint, BindRepoDir[1:], repo.Name) 43 | if !PathExists(tgt) { 44 | if err := os.MkdirAll(tgt, 00755); err != nil { 45 | return err 46 | } 47 | } 48 | 49 | // BindMount the directory into place 50 | if err := mman.BindMount(repo.URI, tgt); err != nil { 51 | return err 52 | } 53 | o.ExtraMounts = append(o.ExtraMounts, tgt) 54 | 55 | // Attempt to autoindex the repo 56 | if repo.AutoIndex { 57 | log.WithFields(log.Fields{ 58 | "name": repo.Name, 59 | }).Debug("Reindexing repository") 60 | 61 | command := fmt.Sprintf("cd %s/%s; %s", BindRepoDir, repo.Name, eopkgCommand("eopkg index --skip-signing .")) 62 | err := ChrootExec(notif, o.MountPoint, command) 63 | notif.SetActivePID(0) 64 | if err != nil { 65 | return err 66 | } 67 | } else { 68 | tgtIndex := filepath.Join(tgt, "eopkg-index.xml.xz") 69 | if !PathExists(tgtIndex) { 70 | log.WithFields(log.Fields{ 71 | "name": repo.Name, 72 | }).Warning("Repository index doesn't exist. Please index it to use it") 73 | } 74 | } 75 | 76 | // Now add the local repo 77 | chrootLocal := filepath.Join(BindRepoDir, repo.Name, "eopkg-index.xml.xz") 78 | return pkgManager.AddRepo(repo.Name, chrootLocal) 79 | } 80 | 81 | func (p *Package) removeRepos(pkgManager *EopkgManager, repos []string) error { 82 | if len(repos) < 1 { 83 | return nil 84 | } 85 | for _, id := range repos { 86 | log.WithFields(log.Fields{ 87 | "name": id, 88 | }).Debug("Removing repository") 89 | if err := pkgManager.RemoveRepo(id); err != nil { 90 | log.WithFields(log.Fields{ 91 | "error": err, 92 | "name": id, 93 | }).Error("Failed to remove repository") 94 | return err 95 | } 96 | } 97 | return nil 98 | } 99 | 100 | // addRepos will add the specified filtered set of repos to the rootfs 101 | func (p *Package) addRepos(notif PidNotifier, o *Overlay, pkgManager *EopkgManager, repos []*Repo) error { 102 | if len(repos) < 1 { 103 | return nil 104 | } 105 | for _, repo := range repos { 106 | if repo.Local { 107 | log.WithFields(log.Fields{ 108 | "name": repo.Name, 109 | "path": repo.URI, 110 | }).Debug("Adding local repo to system") 111 | 112 | if err := p.addLocalRepo(notif, o, pkgManager, repo); err != nil { 113 | log.WithFields(log.Fields{ 114 | "name": repo.Name, 115 | "error": err, 116 | }).Error("Failed to add local repo to system") 117 | return err 118 | } 119 | continue 120 | } 121 | log.WithFields(log.Fields{ 122 | "name": repo.Name, 123 | "url": repo.URI, 124 | }).Debug("Adding repo to system") 125 | if err := pkgManager.AddRepo(repo.Name, repo.URI); err != nil { 126 | log.WithFields(log.Fields{ 127 | "error": err, 128 | "name": repo.Name, 129 | }).Error("Failed to add repo to system") 130 | return err 131 | } 132 | } 133 | return nil 134 | } 135 | 136 | // ConfigureRepos will attempt to configure the repos according to the configuration 137 | // of the manager. 138 | func (p *Package) ConfigureRepos(notif PidNotifier, o *Overlay, pkgManager *EopkgManager, profile *Profile) error { 139 | repos, err := pkgManager.GetRepos() 140 | if err != nil { 141 | return err 142 | } 143 | 144 | var removals []string 145 | 146 | // Find out which repos to remove 147 | if len(profile.RemoveRepos) == 1 && profile.RemoveRepos[0] == "*" { 148 | for _, r := range repos { 149 | removals = append(removals, r.ID) 150 | } 151 | } else { 152 | for _, r := range profile.RemoveRepos { 153 | removals = append(removals, r) 154 | } 155 | } 156 | 157 | if err := p.removeRepos(pkgManager, removals); err != nil { 158 | return err 159 | } 160 | 161 | var addRepos []*Repo 162 | 163 | if (len(profile.AddRepos) == 1 && profile.AddRepos[0] == "*") || len(profile.AddRepos) == 0 { 164 | for _, repo := range profile.Repos { 165 | addRepos = append(addRepos, repo) 166 | } 167 | } else { 168 | for _, id := range profile.AddRepos { 169 | addRepos = append(addRepos, profile.Repos[id]) 170 | } 171 | } 172 | 173 | return p.addRepos(notif, o, pkgManager, addRepos) 174 | } 175 | -------------------------------------------------------------------------------- /src/builder/source/git.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2016-2017 Ikey Doherty 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package source 18 | 19 | import ( 20 | "errors" 21 | "fmt" 22 | log "github.com/Sirupsen/logrus" 23 | "github.com/libgit2/git2go" 24 | "github.com/solus-project/libosdev/commands" 25 | "net/url" 26 | "os" 27 | "path/filepath" 28 | "strings" 29 | ) 30 | 31 | const ( 32 | // GitSourceDir is the base directory for all cached git sources 33 | GitSourceDir = "/var/lib/solbuild/sources/git" 34 | ) 35 | 36 | var ( 37 | // ErrGitNoContinue is returned when git processing cannot continue 38 | ErrGitNoContinue = errors.New("Fatal errors in git fetch") 39 | ) 40 | 41 | // A GitSource as referenced by `ypkg` build spec. A git source must have 42 | // a valid ref to check out to. 43 | type GitSource struct { 44 | URI string 45 | Ref string 46 | BaseName string 47 | ClonePath string // This is where we will have cloned into 48 | } 49 | 50 | // NewGit will create a new GitSource for the given URI & ref combination. 51 | func NewGit(uri, ref string) (*GitSource, error) { 52 | // Ensure we have a valid URL first. 53 | urlObj, err := url.Parse(uri) 54 | if err != nil { 55 | return nil, err 56 | } 57 | 58 | bs := filepath.Base(urlObj.Path) 59 | if !strings.HasSuffix(bs, ".git") { 60 | bs += ".git" 61 | } 62 | 63 | // This is where we intend to clone to locally 64 | clonePath := filepath.Join(GitSourceDir, urlObj.Host, filepath.Dir(urlObj.Path), bs) 65 | 66 | g := &GitSource{ 67 | URI: uri, 68 | Ref: ref, 69 | BaseName: bs, 70 | ClonePath: clonePath, 71 | } 72 | 73 | return g, nil 74 | } 75 | 76 | // completed is called when the fetch is done 77 | func (g *GitSource) completed(r git.RemoteCompletion) git.ErrorCode { 78 | log.WithFields(log.Fields{ 79 | "source": g.BaseName, 80 | }).Debug("Completed fetch of git source") 81 | return 0 82 | } 83 | 84 | // message will be called to emit standard git text to the terminal 85 | func (g *GitSource) message(str string) git.ErrorCode { 86 | os.Stdout.Write([]byte(str)) 87 | return 0 88 | } 89 | 90 | // CreateCallbacks will create the default git callbacks 91 | func (g *GitSource) CreateCallbacks() git.RemoteCallbacks { 92 | return git.RemoteCallbacks{ 93 | SidebandProgressCallback: g.message, 94 | } 95 | } 96 | 97 | // Clone will set do a bare mirror clone of the remote repo to the local 98 | // cache. 99 | func (g *GitSource) Clone() error { 100 | // Attempt cloning 101 | log.WithFields(log.Fields{ 102 | "uri": g.URI, 103 | }).Debug("Cloning git source") 104 | 105 | fetchOpts := &git.FetchOptions{ 106 | RemoteCallbacks: g.CreateCallbacks(), 107 | } 108 | 109 | _, err := git.Clone(g.URI, g.ClonePath, &git.CloneOptions{ 110 | Bare: false, 111 | FetchOptions: fetchOpts, 112 | }) 113 | return err 114 | } 115 | 116 | // HasTag will attempt to find the tag, if possible 117 | func (g *GitSource) HasTag(repo *git.Repository, tagName string) bool { 118 | haveTag := false 119 | repo.Tags.Foreach(func(name string, id *git.Oid) error { 120 | if name == "refs/tags/"+tagName { 121 | haveTag = true 122 | } 123 | return nil 124 | }) 125 | return haveTag 126 | } 127 | 128 | // fetch will attempt 129 | func (g *GitSource) fetch(repo *git.Repository) error { 130 | log.WithFields(log.Fields{ 131 | "uri": g.URI, 132 | }).Info("Git fetching existing clone") 133 | remote, err := repo.Remotes.Lookup("origin") 134 | if err != nil { 135 | log.WithFields(log.Fields{ 136 | "error": err, 137 | "uri": g.URI, 138 | }).Error("Failed to find git remote") 139 | return err 140 | } 141 | 142 | fetchOpts := &git.FetchOptions{ 143 | RemoteCallbacks: g.CreateCallbacks(), 144 | } 145 | 146 | return remote.Fetch([]string{}, fetchOpts, "") 147 | } 148 | 149 | // GetCommitID will attempt to find the oid of the selected ref type 150 | func (g *GitSource) GetCommitID(repo *git.Repository) string { 151 | oid := "" 152 | // Attempt to find the branch 153 | branch, err := repo.LookupBranch(g.Ref, git.BranchAll) 154 | if err == nil { 155 | oid = branch.Target().String() 156 | log.WithFields(log.Fields{ 157 | "branch": g.Ref, 158 | "sha": oid, 159 | }).Debug("Found git commit of branch") 160 | return oid 161 | } 162 | 163 | tagName := g.Ref 164 | if !strings.HasPrefix(tagName, "refs/tags") { 165 | tagName = "refs/tags/" + tagName 166 | } 167 | 168 | repo.Tags.Foreach(func(name string, id *git.Oid) error { 169 | if name == tagName { 170 | oid = id.String() 171 | // Force break the foreach 172 | return errors.New("") 173 | } 174 | return nil 175 | }) 176 | 177 | // Tag set the oid 178 | if oid != "" { 179 | log.WithFields(log.Fields{ 180 | "tag": tagName, 181 | "sha": oid, 182 | }).Debug("Found git commit of tag") 183 | return oid 184 | } 185 | 186 | // Check the oid is valid 187 | oid = g.Ref 188 | obj, err := git.NewOid(oid) 189 | if err != nil { 190 | return "" 191 | } 192 | 193 | // Check if its a commit 194 | _, err = repo.Lookup(obj) 195 | if err != nil { 196 | return "" 197 | } 198 | log.WithFields(log.Fields{ 199 | "tag": tagName, 200 | "sha": oid, 201 | }).Debug("Found git commit") 202 | return obj.String() 203 | } 204 | 205 | // GetHead will attempt to gain the OID for head 206 | func (g *GitSource) GetHead(repo *git.Repository) (string, error) { 207 | head, err := repo.Head() 208 | if err != nil { 209 | return "", err 210 | } 211 | return head.Target().String(), nil 212 | } 213 | 214 | // resetOnto will attempt to reset the repo (hard) onto the given commit 215 | func (g *GitSource) resetOnto(repo *git.Repository, ref string) error { 216 | // this stuff _really_ shouldn't happen but oh well. 217 | oid, err := git.NewOid(ref) 218 | if err != nil { 219 | return err 220 | } 221 | commitFind, err := repo.Lookup(oid) 222 | if err != nil { 223 | return err 224 | } 225 | 226 | commitObj, err := commitFind.Peel(git.ObjectCommit) 227 | if err != nil { 228 | return err 229 | } 230 | commit, err := commitObj.AsCommit() 231 | if err != nil { 232 | return err 233 | } 234 | 235 | log.WithFields(log.Fields{ 236 | "sha": ref, 237 | }).Debug("Resetting git repository to commit") 238 | 239 | checkOpts := &git.CheckoutOpts{ 240 | Strategy: git.CheckoutForce | git.CheckoutRemoveUntracked | git.CheckoutRemoveIgnored} 241 | 242 | if err := repo.ResetToCommit(commit, git.ResetHard, checkOpts); err != nil { 243 | log.WithFields(log.Fields{ 244 | "error": err, 245 | "sha": ref, 246 | }).Error("Failed to reset git repository") 247 | return err 248 | } 249 | 250 | return nil 251 | } 252 | 253 | // submodules will handle setup of the git submodules after a 254 | // reset has taken place. 255 | func (g *GitSource) submodules() error { 256 | // IDK What else to tell ya, git2go submodules is broken 257 | cmd := []string{"submodule", "update", "--init", "--recursive"} 258 | return commands.ExecStdoutArgsDir(g.ClonePath, "git", cmd) 259 | } 260 | 261 | // Fetch will attempt to download the git tree locally. If it already exists 262 | // then we'll make an attempt to update it. 263 | func (g *GitSource) Fetch() error { 264 | hadRepo := true 265 | 266 | // First things first, clone if necessary 267 | if !PathExists(g.ClonePath) { 268 | if err := g.Clone(); err != nil { 269 | log.WithFields(log.Fields{ 270 | "error": err, 271 | "uri": g.URI, 272 | }).Error("Failed to clone remote repository") 273 | return err 274 | } 275 | hadRepo = false 276 | } 277 | 278 | // Now open the repo and validate it 279 | repo, err := git.OpenRepository(g.ClonePath) 280 | if err != nil { 281 | return err 282 | } 283 | 284 | wantedCommit := g.GetCommitID(repo) 285 | if wantedCommit == "" { 286 | // Logic here being we just cloned it. Where is it? 287 | if !hadRepo { 288 | return fmt.Errorf("Cannot continue with git processing") 289 | } 290 | // So try to fetch it 291 | if err := g.fetch(repo); err != nil { 292 | return err 293 | } 294 | // Re-establish the wanted commit 295 | wantedCommit = g.GetCommitID(repo) 296 | } 297 | 298 | // Can't proceed now. Just doesn't exist 299 | if wantedCommit == "" { 300 | return ErrGitNoContinue 301 | } 302 | 303 | // Attempt reset 304 | if err := g.resetOnto(repo, wantedCommit); err != nil { 305 | return err 306 | } 307 | 308 | // Check out submodules 309 | return g.submodules() 310 | } 311 | 312 | // IsFetched will check if we have the ref available, if not it will return 313 | // false so that Fetch() can do the hard work. 314 | func (g *GitSource) IsFetched() bool { 315 | return false 316 | } 317 | 318 | // GetBindConfiguration will return a config that enables bind mounting 319 | // the bare git clone from the host side into the container, at which 320 | // point ypkg can git clone from the bare git into a new tree and check 321 | // out, make changes, etc. 322 | func (g *GitSource) GetBindConfiguration(sourcedir string) BindConfiguration { 323 | return BindConfiguration{ 324 | g.ClonePath, 325 | filepath.Join(sourcedir, g.BaseName), 326 | } 327 | } 328 | 329 | // GetIdentifier will return a human readable string to represent this 330 | // git source in the event of errors. 331 | func (g *GitSource) GetIdentifier() string { 332 | return fmt.Sprintf("%s#%s", g.URI, g.Ref) 333 | } 334 | -------------------------------------------------------------------------------- /src/builder/source/main.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2016-2017 Ikey Doherty 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package source 18 | 19 | import ( 20 | "os" 21 | "strings" 22 | ) 23 | 24 | const ( 25 | // SourceDir is where we store all tarballs 26 | SourceDir = "/var/lib/solbuild/sources" 27 | 28 | // SourceStagingDir is where we initially fetch downloads 29 | SourceStagingDir = "/var/lib/solbuild/sources/staging" 30 | ) 31 | 32 | // A BindConfiguration is used by a source as a way to express bind 33 | // mounts required for a given source. 34 | // 35 | // In solbuild, *all* sources are bind mounted to the target cache, 36 | // regardless of their type. 37 | // 38 | // Special care is taken to ensure that they will be bound in a way 39 | // compatible with the target system. 40 | type BindConfiguration struct { 41 | BindSource string // The localy cached source 42 | BindTarget string // Target within the filesystem 43 | } 44 | 45 | // A Source is a general representation of source listed in a package 46 | // spec file. 47 | // 48 | // Source's may be of multiple types, but all are abstracted and dealt 49 | // with by the interfaces. 50 | type Source interface { 51 | 52 | // IsFetched is called during the early build process to determine 53 | // whether this source is available for use. 54 | IsFetched() bool 55 | 56 | // Fetch will attempt to fetch the this source locally and cache it. 57 | Fetch() error 58 | 59 | // GetBindConfiguration should return a valid configuration specifying 60 | // the origin on our local filesystem, and the target within the container. 61 | // The target should include the full source dir. 62 | GetBindConfiguration(rootfs string) BindConfiguration 63 | 64 | // GetIdentifier will return the appropriate representation for a given 65 | // source URL. 66 | GetIdentifier() string 67 | } 68 | 69 | // New will return a new source for the specified URL. 70 | // 71 | // Validator is the value by which the source will be validated, depending 72 | // on implementation. For example, the SimpleSource backend will expect a 73 | // hashsum: sha256sum for package.yml, and sha1sum for legacy. 74 | // 75 | // The legacy argument will determine whether special care should be taken 76 | // for legacy packages (i.e. sha1sum vs sha256sum). 77 | // 78 | // In all cases, New will fallback to the SimpleSource implementation 79 | func New(uri, validator string, legacy bool) (Source, error) { 80 | if legacy { 81 | return NewSimple(uri, validator, legacy) 82 | } 83 | // Handle git sources. Not supported in legacy format, ypkg only. 84 | if strings.HasPrefix(uri, "git|") { 85 | return NewGit(uri[len("git|"):], validator) 86 | } 87 | return NewSimple(uri, validator, legacy) 88 | } 89 | 90 | // PathExists is a helper function to determine the existence of a file path 91 | func PathExists(path string) bool { 92 | if st, err := os.Stat(path); err == nil && st != nil { 93 | return true 94 | } 95 | return false 96 | } 97 | -------------------------------------------------------------------------------- /src/builder/source/simple.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2016-2017 Ikey Doherty 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package source 18 | 19 | import ( 20 | "crypto/sha1" 21 | "crypto/sha256" 22 | "encoding/hex" 23 | "fmt" 24 | log "github.com/Sirupsen/logrus" 25 | curl "github.com/andelf/go-curl" 26 | "github.com/cheggaaa/pb" 27 | "io/ioutil" 28 | "net/url" 29 | "os" 30 | "path/filepath" 31 | ) 32 | 33 | // A SimpleSource is a tarball or other source for a package 34 | type SimpleSource struct { 35 | URI string 36 | File string // Basename of the file 37 | 38 | legacy bool // If this is ypkg or not 39 | validator string // Validation key for this source 40 | 41 | url *url.URL 42 | } 43 | 44 | // NewSimple will create a new source instance 45 | func NewSimple(uri, validator string, legacy bool) (*SimpleSource, error) { 46 | // Ensure the URI is actually valid. 47 | uriObj, err := url.Parse(uri) 48 | if err != nil { 49 | return nil, err 50 | } 51 | ret := &SimpleSource{ 52 | URI: uri, 53 | File: filepath.Base(uriObj.Path), 54 | legacy: legacy, 55 | validator: validator, 56 | url: uriObj, 57 | } 58 | return ret, nil 59 | } 60 | 61 | // GetIdentifier will return the URI associated with this source. 62 | func (s *SimpleSource) GetIdentifier() string { 63 | return s.URI 64 | } 65 | 66 | // GetBindConfiguration will return the pair for binding our tarballs. 67 | func (s *SimpleSource) GetBindConfiguration(rootfs string) BindConfiguration { 68 | return BindConfiguration{ 69 | BindSource: s.GetPath(s.validator), 70 | BindTarget: filepath.Join(rootfs, s.File), 71 | } 72 | } 73 | 74 | // GetPath gets the path on the filesystem of the source 75 | func (s *SimpleSource) GetPath(hash string) string { 76 | return filepath.Join(SourceDir, hash, s.File) 77 | } 78 | 79 | // GetSHA1Sum will return the sha1sum for the given path 80 | func (s *SimpleSource) GetSHA1Sum(path string) (string, error) { 81 | inp, err := ioutil.ReadFile(path) 82 | if err != nil { 83 | return "", err 84 | } 85 | hash := sha1.New() 86 | hash.Write(inp) 87 | sum := hash.Sum(nil) 88 | return hex.EncodeToString(sum), nil 89 | } 90 | 91 | // GetSHA256Sum will return the sha1sum for the given path 92 | func (s *SimpleSource) GetSHA256Sum(path string) (string, error) { 93 | inp, err := ioutil.ReadFile(path) 94 | if err != nil { 95 | return "", err 96 | } 97 | hash := sha256.New() 98 | hash.Write(inp) 99 | sum := hash.Sum(nil) 100 | return hex.EncodeToString(sum), nil 101 | } 102 | 103 | // IsFetched will determine if the source is already present 104 | func (s *SimpleSource) IsFetched() bool { 105 | return PathExists(s.GetPath(s.validator)) 106 | } 107 | 108 | // download utilises CURL to do all downloads 109 | func (s *SimpleSource) download(destination string) error { 110 | hnd := curl.EasyInit() 111 | defer hnd.Cleanup() 112 | 113 | hnd.Setopt(curl.OPT_URL, s.URI) 114 | hnd.Setopt(curl.OPT_FOLLOWLOCATION, 1) 115 | 116 | out, err := os.Create(destination) 117 | if err != nil { 118 | return err 119 | } 120 | 121 | pbar := pb.New64(0).Prefix(filepath.Base(destination)) 122 | pbar.Set(0) 123 | pbar.SetUnits(pb.U_BYTES) 124 | pbar.SetMaxWidth(80) 125 | pbar.ShowSpeed = true 126 | 127 | writer := func(data []byte, udata interface{}) bool { 128 | if _, err := out.Write(data); err != nil { 129 | return false 130 | } 131 | return true 132 | } 133 | progress := func(total, now, utotal, unow float64, udata interface{}) bool { 134 | pbar.Total = int64(total) 135 | pbar.Set64(int64(now)) 136 | pbar.Update() 137 | return true 138 | } 139 | 140 | hnd.Setopt(curl.OPT_WRITEFUNCTION, writer) 141 | hnd.Setopt(curl.OPT_NOPROGRESS, false) 142 | hnd.Setopt(curl.OPT_PROGRESSFUNCTION, progress) 143 | // Enforce internal 300 second connect timeout in libcurl 144 | hnd.Setopt(curl.OPT_CONNECTTIMEOUT, 0) 145 | hnd.Setopt(curl.OPT_USERAGENT, fmt.Sprintf("solbuild 1.4.2")) 146 | 147 | pbar.Start() 148 | defer func() { 149 | pbar.Update() 150 | pbar.Finish() 151 | }() 152 | 153 | return hnd.Perform() 154 | } 155 | 156 | // Fetch will download the given source and cache it locally 157 | func (s *SimpleSource) Fetch() error { 158 | // Now go and download it 159 | log.WithFields(log.Fields{ 160 | "uri": s.URI, 161 | }).Debug("Downloading source") 162 | 163 | destPath := filepath.Join(SourceStagingDir, s.File) 164 | 165 | // Check staging is available 166 | if !PathExists(SourceStagingDir) { 167 | if err := os.MkdirAll(SourceStagingDir, 00755); err != nil { 168 | return err 169 | } 170 | } 171 | 172 | // Grab the file 173 | if err := s.download(destPath); err != nil { 174 | return err 175 | } 176 | 177 | hash, err := s.GetSHA256Sum(destPath) 178 | if err != nil { 179 | return err 180 | } 181 | 182 | // Make the target directory 183 | tgtDir := filepath.Join(SourceDir, hash) 184 | if !PathExists(tgtDir) { 185 | if err := os.MkdirAll(tgtDir, 00755); err != nil { 186 | return err 187 | } 188 | } 189 | // Move from staging into hash based directory 190 | dest := filepath.Join(tgtDir, s.File) 191 | if err := os.Rename(destPath, dest); err != nil { 192 | return err 193 | } 194 | // If the file has a sha1sum set, symlink it to the sha256sum because 195 | // it's a legacy archive (pspec.xml) 196 | if s.legacy { 197 | sha, err := s.GetSHA1Sum(dest) 198 | if err != nil { 199 | return err 200 | } 201 | tgtLink := filepath.Join(SourceDir, sha) 202 | if err := os.Symlink(hash, tgtLink); err != nil { 203 | return err 204 | } 205 | } 206 | return nil 207 | } 208 | -------------------------------------------------------------------------------- /src/builder/testdata/group: -------------------------------------------------------------------------------- 1 | root:x:0: 2 | bin:x:1: 3 | sys:x:2: 4 | kmem:x:3: 5 | tape:x:4: 6 | tty:x:5: 7 | daemon:x:6: 8 | floppy:x:7: 9 | disk:x:8: 10 | dialout:x:10: 11 | audio:x:11:ikey 12 | video:x:12:ikey 13 | utmp:x:13: 14 | usb:x:14: 15 | cdrom:x:15:ikey 16 | mail:x:34: 17 | nogroup:x:99: 18 | wheel:x:1001: 19 | lock:x:54: 20 | messagebus:x:18: 21 | polkitd:x:27: 22 | mlocate:x:17: 23 | sudo:x:28:ikey,derpmcderpface 24 | adm:x:996: 25 | users:x:998: 26 | systemd-network:x:992: 27 | systemd-resolve:x:993: 28 | nobody:x:65534: 29 | systemd-bus-proxy:x:995: 30 | systemd-timesync:x:994: 31 | systemd-journal:x:997: 32 | input:x:999: 33 | ikey:x:1000: 34 | mock:x:135:ikey 35 | lpadmin:x:19:ikey 36 | gdm:x:21: 37 | vboxusers:x:134: 38 | avahi:x:84: 39 | fuse:x:104: 40 | pulse-access:x:59: 41 | pulse:x:58: 42 | colord:x:124: 43 | derpmcderpface:x:1002: 44 | sshd:x:50: 45 | docker:x:145:ikey 46 | lightdm:x:620: 47 | lp:x:9: 48 | phpfpm:x:991:phpfpm 49 | -------------------------------------------------------------------------------- /src/builder/testdata/passwd: -------------------------------------------------------------------------------- 1 | root:x:0:0:root:/root:/bin/bash 2 | bin:x:1:1:bin:/dev/null:/bin/false 3 | nobody:x:99:99:Unprivileged User:/dev/null:/bin/false 4 | messagebus:x:18:18:D-Bus Message Daemon:/var/run/dbus:/bin/false 5 | polkitd:x:27:27:PolicyKit Daemon:/var/empty:/bin/false 6 | systemd-network:x:992:992:systemd Network Management:/:/sbin/nologin 7 | systemd-timesync:x:994:994:systemd Time Synchronization:/:/sbin/nologin 8 | systemd-bus-proxy:x:995:995:systemd Bus Proxy:/:/sbin/nologin 9 | systemd-resolve:x:993:993:systemd Resolver:/:/sbin/nologin 10 | ikey:x:1000:1000:Ikey Doherty:/home/ikey:/bin/zsh 11 | gdm:x:21:21:GDM:/var/lib/gdm:/bin/false 12 | avahi:x:84:84:Avahi Daemon Owner:/var/run/avahi:/bin/false 13 | pulse:x:58:58:PulseAudio Daemon:/var/run/pulse:/sbin/nologin 14 | colord:x:124:124:colord:/var/lib/colord:/sbin/nologin 15 | derpmcderpface:x:1001:1002::/home/derpmcderpface:/bin/bash 16 | sshd:x:50:50:SSH Daemon:/var/lib/sshd/:/bin/false 17 | lightdm:x:620:620:LightDM:/var/lightdm:/bin/false 18 | lp:x:9:9:Print Service User:/var/spool/cups:/bin/false 19 | phpfpm:x:991:991::/:/sbin/nologin 20 | -------------------------------------------------------------------------------- /src/builder/testdata/unstable.profile: -------------------------------------------------------------------------------- 1 | image = "unstable-x86_64" 2 | 3 | # Remove all the repos from the base image 4 | # remove_repos = ['*'] 5 | 6 | # Remove just a single repo from the base image 7 | # remove_repos = ['Solus'] 8 | 9 | # Restrict enabled repos to just one repo 10 | add_repos = ["Solus"] 11 | 12 | # Example of adding a remote repo 13 | [repo.Solus] 14 | uri = "https://packages.solus-project.com/unstable/eopkg-index.xml.xz" 15 | 16 | # Add a local repository by bind mounting it into chroot on each build 17 | [repo.Local] 18 | uri = "/var/lib/myrepo" 19 | local = true 20 | 21 | # A local repo with automatic indexing 22 | [repo.LocalIndexed] 23 | uri = "/var/lib/myOtherRepo" 24 | local = true 25 | autoindex = true 26 | -------------------------------------------------------------------------------- /src/builder/transit_manifest.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2017 Solus Project 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package builder 18 | 19 | import ( 20 | "bytes" 21 | "errors" 22 | "github.com/BurntSushi/toml" 23 | "io/ioutil" 24 | "path/filepath" 25 | "strings" 26 | ) 27 | 28 | const ( 29 | // TransitManifestSuffix is the extension that a valid transit manifest must have 30 | TransitManifestSuffix = ".tram" 31 | ) 32 | 33 | var ( 34 | // ErrIllegalUpload is returned when someone is a spanner and tries uploading an unsupported file 35 | ErrIllegalUpload = errors.New("The manifest file is NOT an eopkg") 36 | ) 37 | 38 | // A TransitManifestHeader is required in all .tram uploads to ensure that both 39 | // the sender and recipient are talking in the same fashion. 40 | type TransitManifestHeader struct { 41 | // Versioning to protect against future format changes 42 | Version string `toml:"version"` 43 | 44 | // The repo that the uploader is intending to upload *to* 45 | Target string `toml:"target"` 46 | } 47 | 48 | // A TransitManifest is provided by build servers to validate the upload of 49 | // packages into the incoming directory. 50 | // 51 | // This is to ensure all uploads are intentional, complete and verifiable. 52 | type TransitManifest struct { 53 | 54 | // Every .tram file has a [manifest] header - this will never change and is 55 | // version agnostic. 56 | Manifest TransitManifestHeader `toml:"manifest"` 57 | 58 | // A list of files that accompanied this .tram upload 59 | File []TransitManifestFile `toml:"file"` 60 | } 61 | 62 | // TransitManifestFile provides simple verification data for each file in the 63 | // uploaded payload. 64 | type TransitManifestFile struct { 65 | 66 | // Relative filename, i.e. nano-2.7.5-68-1-x86_64.eopkg 67 | Path string `toml:"path"` 68 | 69 | // Cryptographic checksum to allow integrity checks post-upload/pre-merge 70 | Sha256 string `toml:"sha256"` 71 | } 72 | 73 | // NewTransitManifest will attempt to load the transit manifest from the 74 | // named path and perform *basic* validation. 75 | func NewTransitManifest(target string) *TransitManifest { 76 | return &TransitManifest{ 77 | Manifest: TransitManifestHeader{ 78 | Version: "1.0", 79 | Target: target, 80 | }, 81 | } 82 | } 83 | 84 | // AddFile will attempt to add a file to the payload for this package 85 | func (t *TransitManifest) AddFile(path string) error { 86 | if !strings.HasSuffix(path, ".eopkg") { 87 | return ErrIllegalUpload 88 | } 89 | hash, err := FileSha256sum(path) 90 | if err != nil { 91 | return err 92 | } 93 | 94 | t.File = append(t.File, TransitManifestFile{ 95 | Path: filepath.Base(path), 96 | Sha256: hash, 97 | }) 98 | return nil 99 | } 100 | 101 | // Write will dump the manifest to the given file path 102 | func (t *TransitManifest) Write(path string) error { 103 | blob := bytes.Buffer{} 104 | tmenc := toml.NewEncoder(&blob) 105 | // Waste of bytes. 106 | tmenc.Indent = "" 107 | if err := tmenc.Encode(t); err != nil { 108 | return err 109 | } 110 | return ioutil.WriteFile(path, blob.Bytes(), 00644) 111 | } 112 | -------------------------------------------------------------------------------- /src/builder/update.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2016-2017 Ikey Doherty 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package builder 18 | 19 | import ( 20 | log "github.com/Sirupsen/logrus" 21 | "github.com/solus-project/libosdev/disk" 22 | "os" 23 | "path/filepath" 24 | ) 25 | 26 | func (b *BackingImage) updatePackages(notif PidNotifier, pkgManager *EopkgManager) error { 27 | log.Debug("Initialising package manager") 28 | 29 | if err := pkgManager.Init(); err != nil { 30 | log.WithFields(log.Fields{ 31 | "error": err, 32 | }).Error("Failed to initialise package manager") 33 | return err 34 | } 35 | 36 | // Bring up dbus to do Things 37 | log.Debug("Starting D-BUS") 38 | if err := pkgManager.StartDBUS(); err != nil { 39 | log.WithFields(log.Fields{ 40 | "error": err, 41 | }).Error("Failed to start d-bus") 42 | return err 43 | } 44 | 45 | log.Debug("Upgrading builder image") 46 | if err := pkgManager.Upgrade(); err != nil { 47 | log.WithFields(log.Fields{ 48 | "error": err, 49 | }).Error("Failed to perform upgrade") 50 | return err 51 | } 52 | 53 | log.Debug("Asserting system.devel component") 54 | if err := pkgManager.InstallComponent("system.devel"); err != nil { 55 | log.WithFields(log.Fields{ 56 | "error": err, 57 | }).Error("Failed to install system.devel") 58 | return err 59 | } 60 | 61 | // Cleanup now 62 | log.Debug("Stopping D-BUS") 63 | if err := pkgManager.StopDBUS(); err != nil { 64 | log.WithFields(log.Fields{ 65 | "error": err, 66 | }).Error("Failed to stop d-bus") 67 | return err 68 | } 69 | 70 | return nil 71 | } 72 | 73 | // Update will attempt to update the backing image to the latest version 74 | // internally. 75 | func (b *BackingImage) Update(notif PidNotifier, pkgManager *EopkgManager) error { 76 | mountMan := disk.GetMountManager() 77 | log.WithFields(log.Fields{ 78 | "image": b.Name, 79 | }).Debug("Updating backing image") 80 | 81 | if !PathExists(b.RootDir) { 82 | if err := os.MkdirAll(b.RootDir, 00755); err != nil { 83 | log.WithFields(log.Fields{ 84 | "error": err, 85 | }).Error("Failed to create required directories") 86 | return err 87 | } 88 | log.WithFields(log.Fields{ 89 | "dir": b.RootDir, 90 | }).Debug("Created root directory") 91 | } 92 | 93 | log.WithFields(log.Fields{ 94 | "image": b.ImagePath, 95 | "root": b.RootDir, 96 | }).Debug("Mounting rootfs") 97 | 98 | // Mount the rootfs 99 | if err := mountMan.Mount(b.ImagePath, b.RootDir, "auto", "loop"); err != nil { 100 | log.WithFields(log.Fields{ 101 | "image": b.ImagePath, 102 | "error": err, 103 | }).Error("Failed to mount rootfs") 104 | return err 105 | } 106 | 107 | if err := EnsureEopkgLayout(b.RootDir); err != nil { 108 | log.WithFields(log.Fields{ 109 | "image": b.ImagePath, 110 | "error": err, 111 | }).Error("Failed to fix filesystem layout") 112 | return err 113 | } 114 | 115 | procPoint := filepath.Join(b.RootDir, "proc") 116 | 117 | // Bring up proc 118 | log.WithFields(log.Fields{ 119 | "vfs": "/proc", 120 | }).Debug("Mounting vfs") 121 | if err := mountMan.Mount("proc", procPoint, "proc", "nosuid", "noexec"); err != nil { 122 | log.WithFields(log.Fields{ 123 | "error": err, 124 | }).Error("Failed to mount /proc") 125 | return err 126 | } 127 | 128 | // Hand over to package management to do the updates 129 | if err := b.updatePackages(notif, pkgManager); err != nil { 130 | return err 131 | } 132 | 133 | // Lastly, add the user 134 | if err := AddBuildUser(b.RootDir); err != nil { 135 | return err 136 | } 137 | 138 | log.WithFields(log.Fields{ 139 | "profile": b.Name, 140 | }).Debug("Image successfully updated") 141 | 142 | return nil 143 | } 144 | -------------------------------------------------------------------------------- /src/builder/userinfo.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2016-2017 Ikey Doherty 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package builder 18 | 19 | import ( 20 | "fmt" 21 | log "github.com/Sirupsen/logrus" 22 | "github.com/go-ini/ini" 23 | "os" 24 | "os/user" 25 | "path/filepath" 26 | "strconv" 27 | ) 28 | 29 | // UserInfo is required for ypkg builds, to set the .solus/package internally 30 | // and propagate the author details. 31 | type UserInfo struct { 32 | Name string // Actual name 33 | Email string // Actual email 34 | UID int // Unix User Id 35 | GID int // Unix Group ID 36 | HomeDir string // Home directory of the user 37 | Username string // Textual username 38 | } 39 | 40 | const ( 41 | // FallbackUserName is what we fallback to if everything else fails 42 | FallbackUserName = "Automated Package Build" 43 | 44 | // FallbackUserEmail is what we fallback to if everything else fails 45 | FallbackUserEmail = "no.email.set.in.config" 46 | ) 47 | 48 | // SetFromSudo will attempt to set our details from sudo user environment 49 | func (u *UserInfo) SetFromSudo() bool { 50 | sudoUID := os.Getenv("SUDO_UID") 51 | sudoGID := os.Getenv("SUDO_GID") 52 | uid := -1 53 | gid := -1 54 | var err error 55 | 56 | if sudoGID == "" { 57 | sudoGID = sudoUID 58 | } 59 | 60 | if sudoUID == "" { 61 | return false 62 | } 63 | 64 | if uid, err = strconv.Atoi(sudoUID); err != nil { 65 | log.WithFields(log.Fields{ 66 | "error": err, 67 | "uid": sudoUID, 68 | }).Error("Malformed SUDO_UID in environment") 69 | return false 70 | } 71 | 72 | if gid, err = strconv.Atoi(sudoGID); err != nil { 73 | log.WithFields(log.Fields{ 74 | "error": err, 75 | "gid": sudoGID, 76 | }).Error("Malformed SUDO_GID in environment") 77 | return false 78 | } 79 | 80 | u.UID = uid 81 | u.GID = gid 82 | 83 | // Try to set the home directory 84 | usr, err := user.LookupId(sudoUID) 85 | if err != nil { 86 | log.WithFields(log.Fields{ 87 | "error": err, 88 | "uid": uid, 89 | }).Error("Failed to lookup SUDO_USER entry") 90 | return false 91 | } 92 | 93 | // Now store the home directory for that user 94 | u.HomeDir = usr.HomeDir 95 | u.Username = usr.Username 96 | // In case of future fails 97 | u.Name = usr.Name 98 | 99 | return true 100 | } 101 | 102 | // SetFromCurrent will set the UserInfo details from the current user 103 | func (u *UserInfo) SetFromCurrent() { 104 | u.UID = os.Getuid() 105 | u.GID = os.Getgid() 106 | 107 | if usr, err := user.Current(); err != nil { 108 | u.HomeDir = usr.HomeDir 109 | u.Username = usr.Username 110 | u.Name = usr.Name 111 | } else { 112 | log.WithFields(log.Fields{ 113 | "error": err, 114 | "uid": u.UID, 115 | }).Error("Failed to lookup current user") 116 | u.Username = os.Getenv("USERNAME") 117 | u.Name = u.Username 118 | u.HomeDir = filepath.Join("/home", u.Username) 119 | } 120 | } 121 | 122 | // SetFromPackager will set the username/email fields from the legacy solus 123 | // packager file. 124 | func (u *UserInfo) SetFromPackager() bool { 125 | candidatePaths := []string{ 126 | filepath.Join(u.HomeDir, ".solus", "packager"), 127 | filepath.Join(u.HomeDir, ".evolveos", "packager"), 128 | } 129 | 130 | // Attempt to parse one of the packager files 131 | for _, p := range candidatePaths { 132 | if !PathExists(p) { 133 | continue 134 | } 135 | cfg, err := ini.Load(p) 136 | if err != nil { 137 | log.WithFields(log.Fields{ 138 | "error": err, 139 | "path": p, 140 | }).Error("Error loading INI file") 141 | continue 142 | } 143 | 144 | section, err := cfg.GetSection("Packager") 145 | if err != nil { 146 | log.WithFields(log.Fields{ 147 | "path": p, 148 | }).Error("Missing [Packager] section in file") 149 | continue 150 | } 151 | 152 | uname, err := section.GetKey("Name") 153 | if err != nil { 154 | log.WithFields(log.Fields{ 155 | "error": err, 156 | "path": p, 157 | }).Error("Packager file has missing Name") 158 | continue 159 | } 160 | email, err := section.GetKey("Email") 161 | if err != nil { 162 | log.WithFields(log.Fields{ 163 | "error": err, 164 | "path": p, 165 | }).Error("Packager file has missing Email") 166 | continue 167 | } 168 | u.Name = uname.String() 169 | u.Email = email.String() 170 | log.Debug("Setting packager details from packager INI file") 171 | return true 172 | } 173 | 174 | return false 175 | } 176 | 177 | // SetFromGit will set the username/email fields from the git config file 178 | func (u *UserInfo) SetFromGit() bool { 179 | gitConfPath := filepath.Join(u.HomeDir, ".gitconfig") 180 | if !PathExists(gitConfPath) { 181 | return false 182 | } 183 | 184 | cfg, err := ini.Load(gitConfPath) 185 | if err != nil { 186 | log.WithFields(log.Fields{ 187 | "error": err, 188 | "path": gitConfPath, 189 | }).Error("Error loading gitconfig") 190 | return false 191 | } 192 | 193 | section, err := cfg.GetSection("user") 194 | if err != nil { 195 | log.WithFields(log.Fields{ 196 | "path": gitConfPath, 197 | }).Error("Missing [user] section in gitconfig") 198 | return false 199 | } 200 | 201 | uname, err := section.GetKey("name") 202 | if err != nil { 203 | log.WithFields(log.Fields{ 204 | "error": err, 205 | "path": gitConfPath, 206 | }).Error("gitconfig file has missing name") 207 | return false 208 | } 209 | email, err := section.GetKey("email") 210 | if err != nil { 211 | log.WithFields(log.Fields{ 212 | "error": err, 213 | "path": gitConfPath, 214 | }).Error("gitconfig file has missing email") 215 | return false 216 | } 217 | u.Name = uname.String() 218 | u.Email = email.String() 219 | log.Debug("Setting packager details from git config") 220 | 221 | return true 222 | } 223 | 224 | // GetUserInfo will always succeed, as it will use a fallback policy until it 225 | // finally comes up with a valid combination of name/email to use. 226 | func GetUserInfo() *UserInfo { 227 | uinfo := &UserInfo{} 228 | 229 | // First up try to set the uid/gid 230 | if !uinfo.SetFromSudo() { 231 | uinfo.SetFromCurrent() 232 | } 233 | 234 | attempts := []func() bool{ 235 | uinfo.SetFromPackager, 236 | uinfo.SetFromGit, 237 | } 238 | 239 | for _, a := range attempts { 240 | if a() { 241 | return uinfo 242 | } 243 | } 244 | 245 | if uinfo.Name == "" { 246 | uinfo.Name = FallbackUserName 247 | } 248 | if uinfo.Email == "" { 249 | if ho, err := os.Hostname(); err != nil { 250 | uinfo.Email = fmt.Sprintf("%s@%s", uinfo.Username, ho) 251 | } else { 252 | uinfo.Email = FallbackUserEmail 253 | } 254 | } 255 | 256 | return uinfo 257 | } 258 | 259 | // WritePackager will attempt to write the packager file to given path 260 | func (u *UserInfo) WritePackager(path string) error { 261 | fi, err := os.Create(path) 262 | if err != nil { 263 | return err 264 | } 265 | defer fi.Close() 266 | contents := fmt.Sprintf("[Packager]\nName=%s\nEmail=%s\n", u.Name, u.Email) 267 | if _, err := fi.WriteString(contents); err != nil { 268 | return err 269 | } 270 | return nil 271 | } 272 | -------------------------------------------------------------------------------- /src/builder/users.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2016-2017 Ikey Doherty 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package builder 18 | 19 | import ( 20 | "bufio" 21 | "fmt" 22 | "os" 23 | "path/filepath" 24 | "strconv" 25 | "strings" 26 | ) 27 | 28 | // A User is an /etc/passwd defined user 29 | type User struct { 30 | Name string // User Name 31 | UID int // User ID 32 | GID int // User primary Group ID 33 | Gecos string // User Gecos (Pretty name) 34 | Home string // User home directory 35 | Shell string // User shell program 36 | } 37 | 38 | // A Group is an /etc/group defined user 39 | type Group struct { 40 | Name string // Group Name 41 | ID int // Group ID 42 | Members []string // Names of users in group 43 | } 44 | 45 | // Passwd is a simple helper to parse passwd files from a chroot 46 | type Passwd struct { 47 | Users map[string]*User 48 | Groups map[string]*Group 49 | } 50 | 51 | // NewPasswd will parse the given path and return a friendly representation 52 | // of those files 53 | func NewPasswd(path string) (*Passwd, error) { 54 | passwdPath := filepath.Join(path, "passwd") 55 | groupPath := filepath.Join(path, "group") 56 | 57 | var err error 58 | 59 | ret := &Passwd{} 60 | if ret.Users, err = ParseUsers(passwdPath); err != nil { 61 | return nil, err 62 | } 63 | if ret.Groups, err = ParseGroups(groupPath); err != nil { 64 | return nil, err 65 | } 66 | return ret, nil 67 | } 68 | 69 | // ParseUsers will attempt to parse a *NIX style passwd file 70 | func ParseUsers(passwd string) (map[string]*User, error) { 71 | fi, err := os.Open(passwd) 72 | if err != nil { 73 | return nil, err 74 | } 75 | defer fi.Close() 76 | 77 | ret := make(map[string]*User) 78 | 79 | sc := bufio.NewScanner(fi) 80 | for sc.Scan() { 81 | line := sc.Text() 82 | splits := strings.Split(line, ":") 83 | if len(splits) != 7 { 84 | return nil, fmt.Errorf("Invalid number of fields in passwd file: %d", len(splits)) 85 | } 86 | user := &User{ 87 | Name: strings.TrimSpace(splits[0]), 88 | Gecos: strings.TrimSpace(splits[4]), 89 | Home: strings.TrimSpace(splits[5]), 90 | Shell: strings.TrimSpace(splits[6]), 91 | } 92 | // Parse the uid/gid 93 | if uid, err := strconv.Atoi(strings.TrimSpace(splits[2])); err == nil { 94 | user.UID = uid 95 | } else { 96 | return nil, err 97 | } 98 | if gid, err := strconv.Atoi(strings.TrimSpace(splits[3])); err == nil { 99 | user.GID = gid 100 | } else { 101 | return nil, err 102 | } 103 | // Success 104 | ret[user.Name] = user 105 | } 106 | if err := sc.Err(); err != nil { 107 | return nil, err 108 | } 109 | return ret, nil 110 | } 111 | 112 | // ParseGroups will attempt to parse a *NIX style group file 113 | func ParseGroups(grps string) (map[string]*Group, error) { 114 | fi, err := os.Open(grps) 115 | if err != nil { 116 | return nil, err 117 | } 118 | defer fi.Close() 119 | 120 | ret := make(map[string]*Group) 121 | 122 | sc := bufio.NewScanner(fi) 123 | for sc.Scan() { 124 | line := sc.Text() 125 | splits := strings.Split(line, ":") 126 | if len(splits) != 4 { 127 | return nil, fmt.Errorf("Invalid number of fields in group file: %d", len(splits)) 128 | } 129 | group := &Group{ 130 | Name: strings.TrimSpace(splits[0]), 131 | } 132 | // So we don't get one empty member situation 133 | membs := strings.TrimSpace(splits[3]) 134 | if membs != "" { 135 | group.Members = strings.Split(membs, ",") 136 | } 137 | // Parse the gid 138 | if gid, err := strconv.Atoi(strings.TrimSpace(splits[2])); err == nil { 139 | group.ID = gid 140 | } else { 141 | return nil, err 142 | } 143 | // Success 144 | ret[group.Name] = group 145 | } 146 | if err := sc.Err(); err != nil { 147 | return nil, err 148 | } 149 | return ret, nil 150 | } 151 | -------------------------------------------------------------------------------- /src/builder/users_test.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2016-2017 Ikey Doherty 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package builder 18 | 19 | import ( 20 | "strings" 21 | "testing" 22 | ) 23 | 24 | func TestPasswd(t *testing.T) { 25 | if _, err := NewPasswd("./@W'el@@"); err == nil { 26 | t.Fatalf("Should not be able to parse non existent file") 27 | } 28 | 29 | pwd, err := NewPasswd("testdata") 30 | if err != nil { 31 | t.Fatalf("Unable to parse known good passwd data: %v", err) 32 | } 33 | if len(pwd.Users) != 19 { 34 | t.Fatalf("Invalid number of users parsed: %v vs expected 19", len(pwd.Users)) 35 | } 36 | if len(pwd.Groups) != 48 { 37 | t.Fatalf("Invalid number of groups parsed: %v vs expected 48", len(pwd.Groups)) 38 | } 39 | 40 | derp, foundDerp := pwd.Users["derpmcderpface"] 41 | if !foundDerp { 42 | t.Fatalf("Failed to find known user") 43 | } 44 | if derp.UID != 1001 { 45 | t.Fatalf("User ID wrong: %d vs expected 1001", derp.UID) 46 | } 47 | if derp.GID != 1002 { 48 | t.Fatalf("User GID wrong: %d vs expected 1002", derp.UID) 49 | } 50 | if derp.Home != "/home/derpmcderpface" { 51 | t.Fatalf("Wrong homedir: '%s' vs expected /home/derpmcderpface", derp.Home) 52 | } 53 | 54 | if shell := pwd.Users["root"].Shell; shell != "/bin/bash" { 55 | t.Fatalf("Wrong shell for root: %s", shell) 56 | } 57 | 58 | sudo, foundSudo := pwd.Groups["sudo"] 59 | if !foundSudo { 60 | t.Fatalf("I am without sudo") 61 | } 62 | if len(sudo.Members) != 2 { 63 | t.Fatalf("sudo has wrong member count of %d vs expected 2", len(sudo.Members)) 64 | } 65 | members := strings.Join(sudo.Members, ",") 66 | if members != "ikey,derpmcderpface" { 67 | t.Fatalf("Wrong members for sudo: %s", members) 68 | } 69 | 70 | lightdm := pwd.Groups["lightdm"] 71 | if lightdm.ID != pwd.Users["lightdm"].GID { 72 | t.Fatalf("Wrong GID for lightdm: %d vs expected %d", lightdm.ID, pwd.Users["lightdm"].GID) 73 | } 74 | 75 | if len(lightdm.Members) != 0 { 76 | t.Fatalf("Myseriously have members of lightdm: |%s| %d", strings.Join(lightdm.Members, ", "), len(lightdm.Members)) 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /src/builder/util.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2016-2017 Ikey Doherty 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package builder 18 | 19 | import ( 20 | "crypto/sha256" 21 | "encoding/hex" 22 | "fmt" 23 | log "github.com/Sirupsen/logrus" 24 | "github.com/solus-project/libosdev/commands" 25 | "github.com/solus-project/libosdev/disk" 26 | "io/ioutil" 27 | "os" 28 | "os/exec" 29 | "path/filepath" 30 | "strconv" 31 | "strings" 32 | "syscall" 33 | "time" 34 | ) 35 | 36 | var ( 37 | // ChrootEnvironment is the env used by ChrootExec calls 38 | ChrootEnvironment []string 39 | ) 40 | 41 | func init() { 42 | ChrootEnvironment = nil 43 | } 44 | 45 | // PidNotifier provides a simple way to set the PID on a blocking process 46 | type PidNotifier interface { 47 | SetActivePID(int) 48 | } 49 | 50 | // ActivateRoot will do the hard work of actually bring up the overlayfs 51 | // system to allow manipulation of the roots for builds, etc. 52 | func (p *Package) ActivateRoot(overlay *Overlay) error { 53 | log.Debug("Configuring overlay storage") 54 | 55 | // Now mount the overlayfs 56 | if err := overlay.Mount(); err != nil { 57 | return err 58 | } 59 | 60 | // Add build user 61 | if p.Type == PackageTypeYpkg { 62 | if err := AddBuildUser(overlay.MountPoint); err != nil { 63 | return err 64 | } 65 | } 66 | 67 | log.Debug("Bringing up virtual filesystems") 68 | return overlay.MountVFS() 69 | } 70 | 71 | // DeactivateRoot will tear down the previously activated root 72 | func (p *Package) DeactivateRoot(overlay *Overlay) { 73 | MurderDeathKill(overlay.MountPoint) 74 | mountMan := disk.GetMountManager() 75 | commands.SetStdin(nil) 76 | overlay.Unmount() 77 | log.Debug("Requesting unmount of all remaining mountpoints") 78 | mountMan.UnmountAll() 79 | } 80 | 81 | // MurderDeathKill will find all processes with a root matching the given root 82 | // and set about killing them, to assist in clean closing. 83 | func MurderDeathKill(root string) error { 84 | path, err := filepath.EvalSymlinks(root) 85 | if err != nil { 86 | return err 87 | } 88 | 89 | var files []os.FileInfo 90 | 91 | if files, err = ioutil.ReadDir("/proc"); err != nil { 92 | return err 93 | } 94 | 95 | for _, f := range files { 96 | fpath := filepath.Join("/proc", f.Name(), "cwd") 97 | 98 | spath, err := filepath.EvalSymlinks(fpath) 99 | if err != nil { 100 | continue 101 | } 102 | 103 | if spath != path { 104 | continue 105 | } 106 | 107 | spid := f.Name() 108 | var pid int 109 | 110 | if pid, err = strconv.Atoi(spid); err != nil { 111 | log.WithFields(log.Fields{ 112 | "pid": spid, 113 | "error": err, 114 | }).Error("POSIX Weeps - broken pid identifier") 115 | return err 116 | } 117 | 118 | log.WithFields(log.Fields{ 119 | "pid": pid, 120 | }).Debug("Killing child process in chroot") 121 | 122 | if err := syscall.Kill(pid, syscall.SIGTERM); err != nil { 123 | log.WithFields(log.Fields{ 124 | "pid": pid, 125 | }).Error("Error terminating process, attempting force kill") 126 | time.Sleep(400 * time.Millisecond) 127 | if err := syscall.Kill(pid, syscall.SIGKILL); err != nil { 128 | log.WithFields(log.Fields{ 129 | "pid": pid, 130 | }).Error("Error killing (-9) process") 131 | } 132 | } 133 | } 134 | return nil 135 | } 136 | 137 | // TouchFile will create the file if it doesn't exist, enabling use of bind 138 | // mounts. 139 | func TouchFile(path string) error { 140 | w, err := os.OpenFile(path, os.O_RDONLY|os.O_CREATE, 00644) 141 | if err != nil { 142 | return err 143 | } 144 | defer w.Close() 145 | return nil 146 | } 147 | 148 | // SaneEnvironment will generate a clean environment for the chroot'd 149 | // processes to use 150 | func SaneEnvironment(username, home string) []string { 151 | environment := []string{ 152 | "PATH=/usr/bin:/usr/sbin:/bin/:/sbin", 153 | "LANG=en_US.utf8", 154 | "LC_ALL=en_US.utf8", 155 | fmt.Sprintf("HOME=%s", home), 156 | fmt.Sprintf("USER=%s", username), 157 | fmt.Sprintf("USERNAME=%s", username), 158 | } 159 | // Consider an option to even filter these out 160 | permitted := []string{ 161 | "http_proxy", 162 | "https_proxy", 163 | "no_proxy", 164 | "ftp_proxy", 165 | "TERM", 166 | } 167 | if !DisableColors { 168 | permitted = append(permitted, "TERM") 169 | } 170 | for _, p := range permitted { 171 | env := os.Getenv(p) 172 | if env == "" { 173 | p = strings.ToUpper(p) 174 | env = os.Getenv(p) 175 | } 176 | if env == "" { 177 | continue 178 | } 179 | environment = append(environment, 180 | fmt.Sprintf("%s=%s", p, env)) 181 | } 182 | if DisableColors { 183 | environment = append(environment, "TERM=dumb") 184 | } 185 | return environment 186 | } 187 | 188 | // ChrootExec is a simple wrapper to return a correctly set up chroot command, 189 | // so that we can store the PID, for long running tasks 190 | func ChrootExec(notif PidNotifier, dir, command string) error { 191 | args := []string{dir, "/bin/sh", "-c", command} 192 | c := exec.Command("chroot", args...) 193 | c.Stdout = os.Stdout 194 | c.Stderr = os.Stderr 195 | c.Stdin = nil 196 | c.Env = ChrootEnvironment 197 | c.SysProcAttr = &syscall.SysProcAttr{Setsid: true} 198 | 199 | if err := c.Start(); err != nil { 200 | return err 201 | } 202 | notif.SetActivePID(c.Process.Pid) 203 | return c.Wait() 204 | } 205 | 206 | // ChrootExecStdin is almost identical to ChrootExec, except it permits a stdin 207 | // to be associated with the command 208 | func ChrootExecStdin(notif PidNotifier, dir, command string) error { 209 | args := []string{dir, "/bin/sh", "-c", command} 210 | c := exec.Command("chroot", args...) 211 | c.Stdout = os.Stdout 212 | c.Stderr = os.Stderr 213 | c.Stdin = os.Stdin 214 | c.Env = ChrootEnvironment 215 | 216 | if err := c.Start(); err != nil { 217 | return err 218 | } 219 | notif.SetActivePID(c.Process.Pid) 220 | return c.Wait() 221 | } 222 | 223 | // AddBuildUser will attempt to add the solbuild user & group if they've not 224 | // previously been added 225 | // Note this should be changed when Solus goes fully stateless for /etc/passwd 226 | func AddBuildUser(rootfs string) error { 227 | pwd, err := NewPasswd(filepath.Join(rootfs, "etc")) 228 | if err != nil { 229 | log.WithFields(log.Fields{ 230 | "error": err, 231 | }).Error("Unable to discover chroot users") 232 | return err 233 | } 234 | // User already exists 235 | if _, ok := pwd.Users[BuildUser]; ok { 236 | return nil 237 | } 238 | log.WithFields(log.Fields{ 239 | "username": BuildUser, 240 | "uid": BuildUserID, 241 | "gid": BuildUserGID, 242 | "home": BuildUserHome, 243 | "shell": BuildUserShell, 244 | "gecos": BuildUserGecos, 245 | }).Debug("Adding build user to system") 246 | 247 | // Add the build group 248 | if err := commands.AddGroup(rootfs, BuildUser, BuildUserGID); err != nil { 249 | log.WithFields(log.Fields{ 250 | "error": err, 251 | }).Error("Failed to add build group to system") 252 | return err 253 | } 254 | 255 | if err := commands.AddUser(rootfs, BuildUser, BuildUserGecos, BuildUserHome, BuildUserShell, BuildUserID, BuildUserGID); err != nil { 256 | log.WithFields(log.Fields{ 257 | "error": err, 258 | }).Error("Failed to add build user to system") 259 | return err 260 | } 261 | return nil 262 | } 263 | 264 | // FileSha256sum is a quick wrapper to grab the sha256sum for the given file 265 | func FileSha256sum(path string) (string, error) { 266 | mfile, err := MapFile(path) 267 | if err != nil { 268 | return "", err 269 | } 270 | defer mfile.Close() 271 | h := sha256.New() 272 | // Pump from memory into hash for zero-copy sha1sum 273 | h.Write(mfile.Data) 274 | return hex.EncodeToString(h.Sum(nil)), nil 275 | } 276 | -------------------------------------------------------------------------------- /src/solbuild/cmd/build.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2016-2017 Ikey Doherty 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package cmd 18 | 19 | import ( 20 | "builder" 21 | "errors" 22 | "fmt" 23 | log "github.com/Sirupsen/logrus" 24 | "github.com/spf13/cobra" 25 | "os" 26 | "strings" 27 | ) 28 | 29 | var buildCmd = &cobra.Command{ 30 | Use: "build [package.yml|pspec.xml]", 31 | Short: "build a package", 32 | Long: `Build the given package in a chroot environment, and upon success, 33 | store those packages in the current directory`, 34 | RunE: buildPackage, 35 | } 36 | 37 | var tmpfs bool 38 | var tmpfsSize string 39 | var manifest string 40 | 41 | func init() { 42 | buildCmd.Flags().BoolVarP(&tmpfs, "tmpfs", "t", false, "Enable building in a tmpfs") 43 | buildCmd.Flags().StringVarP(&tmpfsSize, "memory", "m", "", "Set the tmpfs size to use") 44 | buildCmd.Flags().StringVarP(&manifest, "transit-manifest", "", "", "Create transit manifest for the given target") 45 | RootCmd.AddCommand(buildCmd) 46 | } 47 | 48 | func buildPackage(cmd *cobra.Command, args []string) error { 49 | pkgPath := "" 50 | 51 | if CLIDebug { 52 | log.SetLevel(log.DebugLevel) 53 | } 54 | log.StandardLogger().Formatter.(*log.TextFormatter).DisableColors = builder.DisableColors 55 | 56 | if len(args) == 1 { 57 | pkgPath = args[0] 58 | } else { 59 | // Try to find the logical path.. 60 | pkgPath = FindLikelyArg() 61 | } 62 | 63 | if os.Geteuid() != 0 { 64 | fmt.Fprintf(os.Stderr, "You must be root to run build packages\n") 65 | os.Exit(1) 66 | } 67 | 68 | // Initialise the build manager 69 | manager, err := builder.NewManager() 70 | if err != nil { 71 | return nil 72 | } 73 | 74 | // Safety first.. 75 | if err = manager.SetProfile(profile); err != nil { 76 | return nil 77 | } 78 | 79 | pkgPath = strings.TrimSpace(pkgPath) 80 | 81 | if pkgPath == "" { 82 | return errors.New("Require a filename to build") 83 | } 84 | 85 | pkg, err := builder.NewPackage(pkgPath) 86 | if err != nil { 87 | fmt.Fprintf(os.Stderr, "Failed to load package: %v\n", err) 88 | return nil 89 | } 90 | 91 | manager.SetManifestTarget(manifest) 92 | 93 | // Set the package 94 | if err := manager.SetPackage(pkg); err != nil { 95 | if err == builder.ErrProfileNotInstalled { 96 | fmt.Fprintf(os.Stderr, "%v: Did you forget to init?\n", err) 97 | } 98 | return nil 99 | } 100 | 101 | manager.SetTmpfs(tmpfs, tmpfsSize) 102 | if err := manager.Build(); err != nil { 103 | log.Error("Failed to build packages") 104 | return nil 105 | } 106 | 107 | log.Info("Building succeeded") 108 | 109 | return nil 110 | } 111 | -------------------------------------------------------------------------------- /src/solbuild/cmd/chroot.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2016-2017 Ikey Doherty 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package cmd 18 | 19 | import ( 20 | "builder" 21 | "errors" 22 | "fmt" 23 | log "github.com/Sirupsen/logrus" 24 | "github.com/spf13/cobra" 25 | "os" 26 | "strings" 27 | ) 28 | 29 | var chrootCmd = &cobra.Command{ 30 | Use: "chroot [package.yml|pspec.xml]", 31 | Short: "chroot into package's build environment", 32 | Long: `Interactively chroot into the package's build environment, to enable 33 | further inspection when issues aren't immediately resolvable, i.e. pkg-config 34 | dependencies.`, 35 | RunE: chrootPackage, 36 | } 37 | 38 | func init() { 39 | RootCmd.AddCommand(chrootCmd) 40 | } 41 | 42 | func chrootPackage(cmd *cobra.Command, args []string) error { 43 | pkgPath := "" 44 | 45 | if CLIDebug { 46 | log.SetLevel(log.DebugLevel) 47 | } 48 | log.StandardLogger().Formatter.(*log.TextFormatter).DisableColors = builder.DisableColors 49 | 50 | if len(args) == 1 { 51 | pkgPath = args[0] 52 | } else { 53 | // Try to find the logical path.. 54 | pkgPath = FindLikelyArg() 55 | } 56 | 57 | pkgPath = strings.TrimSpace(pkgPath) 58 | 59 | if pkgPath == "" { 60 | return errors.New("Require a filename to chroot") 61 | } 62 | 63 | if os.Geteuid() != 0 { 64 | fmt.Fprintf(os.Stderr, "You must be root to use chroot\n") 65 | os.Exit(1) 66 | } 67 | 68 | // Initialise the build manager 69 | manager, err := builder.NewManager() 70 | if err != nil { 71 | return nil 72 | } 73 | // Safety first.. 74 | if err = manager.SetProfile(profile); err != nil { 75 | return nil 76 | } 77 | 78 | pkg, err := builder.NewPackage(pkgPath) 79 | if err != nil { 80 | fmt.Fprintf(os.Stderr, "Failed to load package: %v\n", err) 81 | return nil 82 | } 83 | 84 | // Set the package 85 | if err := manager.SetPackage(pkg); err != nil { 86 | if err == builder.ErrProfileNotInstalled { 87 | fmt.Fprintf(os.Stderr, "%v: Did you forget to init?\n", err) 88 | } 89 | return nil 90 | } 91 | 92 | if err := manager.Chroot(); err != nil { 93 | log.Error("Chroot failure") 94 | return nil 95 | } 96 | 97 | log.Info("Chroot complete") 98 | return nil 99 | } 100 | -------------------------------------------------------------------------------- /src/solbuild/cmd/delete_cache.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2016-2017 Ikey Doherty 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package cmd 18 | 19 | import ( 20 | "builder" 21 | "builder/source" 22 | "fmt" 23 | log "github.com/Sirupsen/logrus" 24 | "github.com/spf13/cobra" 25 | "os" 26 | "strings" 27 | ) 28 | 29 | var deleteCacheCmd = &cobra.Command{ 30 | Use: "delete-cache", 31 | Short: "delete solbuild cached files", 32 | Long: `Delete assets stored on disk by solbuild`, 33 | Aliases: []string{"dc"}, 34 | Run: deleteCache, 35 | } 36 | 37 | // Whether we nuke *all* assets, i.e. sources too 38 | var purgeAll bool 39 | 40 | func init() { 41 | deleteCacheCmd.Flags().BoolVarP(&purgeAll, "all", "a", false, "Also delete ccache, packages and sources") 42 | RootCmd.AddCommand(deleteCacheCmd) 43 | } 44 | 45 | func deleteCache(cmd *cobra.Command, args []string) { 46 | if len(args) == 1 { 47 | profile = strings.TrimSpace(args[0]) 48 | } 49 | 50 | if CLIDebug { 51 | log.SetLevel(log.DebugLevel) 52 | } 53 | log.StandardLogger().Formatter.(*log.TextFormatter).DisableColors = builder.DisableColors 54 | 55 | if os.Geteuid() != 0 { 56 | fmt.Fprintf(os.Stderr, "You must be root to delete caches\n") 57 | os.Exit(1) 58 | } 59 | 60 | // By default include /var/lib/solbuild 61 | nukeDirs := []string{ 62 | builder.OverlayRootDir, 63 | } 64 | 65 | if purgeAll { 66 | nukeDirs = append(nukeDirs, []string{ 67 | builder.CcacheDirectory, 68 | builder.LegacyCcacheDirectory, 69 | builder.PackageCacheDirectory, 70 | source.SourceDir, 71 | }...) 72 | } 73 | 74 | for _, p := range nukeDirs { 75 | if !builder.PathExists(p) { 76 | continue 77 | } 78 | log.WithFields(log.Fields{ 79 | "dir": p, 80 | }).Info("Removing cache directory") 81 | if err := os.RemoveAll(p); err != nil { 82 | log.WithFields(log.Fields{ 83 | "error": err, 84 | "dir": p, 85 | }).Error("Could not remove cache directory") 86 | os.Exit(1) 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /src/solbuild/cmd/index.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2016-2017 Ikey Doherty 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package cmd 18 | 19 | import ( 20 | "builder" 21 | "fmt" 22 | log "github.com/Sirupsen/logrus" 23 | "github.com/spf13/cobra" 24 | "os" 25 | "strings" 26 | ) 27 | 28 | var indexCmd = &cobra.Command{ 29 | Use: "index [directory]", 30 | Short: "create repo index in the given directory", 31 | Long: `Use the given build profile to construct a repository index in the 32 | given directory. If a directory is not specified, then the current directory 33 | is used. This directory will be mounted inside the container and the Solus 34 | machinery will be used to create a repository.`, 35 | RunE: indexPackages, 36 | } 37 | 38 | func init() { 39 | indexCmd.Flags().BoolVarP(&tmpfs, "tmpfs", "t", false, "Enable building in a tmpfs") 40 | indexCmd.Flags().StringVarP(&tmpfsSize, "memory", "m", "", "Set the tmpfs size to use") 41 | RootCmd.AddCommand(indexCmd) 42 | } 43 | 44 | func indexPackages(cmd *cobra.Command, args []string) error { 45 | if CLIDebug { 46 | log.SetLevel(log.DebugLevel) 47 | } 48 | log.StandardLogger().Formatter.(*log.TextFormatter).DisableColors = builder.DisableColors 49 | 50 | if os.Geteuid() != 0 { 51 | fmt.Fprintf(os.Stderr, "You must be root to use index\n") 52 | os.Exit(1) 53 | } 54 | 55 | indexDir := "." 56 | if len(args) == 1 { 57 | indexDir = args[0] 58 | } 59 | 60 | indexDir = strings.TrimSpace(indexDir) 61 | 62 | // Initialise the build manager 63 | manager, err := builder.NewManager() 64 | if err != nil { 65 | return nil 66 | } 67 | // Safety first.. 68 | if err = manager.SetProfile(profile); err != nil { 69 | return nil 70 | } 71 | 72 | // Set the package 73 | if err := manager.SetPackage(&builder.IndexPackage); err != nil { 74 | if err == builder.ErrProfileNotInstalled { 75 | fmt.Fprintf(os.Stderr, "%v: Did you forget to init?\n", err) 76 | } 77 | return nil 78 | } 79 | 80 | manager.SetTmpfs(tmpfs, tmpfsSize) 81 | if err := manager.Index(indexDir); err != nil { 82 | log.Error("Index failure") 83 | return nil 84 | } 85 | 86 | log.Info("Indexing complete") 87 | return nil 88 | } 89 | -------------------------------------------------------------------------------- /src/solbuild/cmd/init.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2016-2017 Ikey Doherty 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package cmd 18 | 19 | import ( 20 | "builder" 21 | "fmt" 22 | "io" 23 | "net/http" 24 | "os" 25 | "strings" 26 | 27 | log "github.com/Sirupsen/logrus" 28 | "github.com/cheggaaa/pb" 29 | "github.com/solus-project/libosdev/commands" 30 | "github.com/spf13/cobra" 31 | ) 32 | 33 | var ( 34 | initCmd = &cobra.Command{ 35 | Use: "init [profile]", 36 | Short: "initialise a solbuild profile", 37 | Long: `Initialise a solbuild profile so that it can be used for subsequent 38 | builds`, 39 | Run: initProfile, 40 | } 41 | 42 | // Whether we should automatically update the image after initialising it. 43 | autoUpdate bool 44 | ) 45 | 46 | func init() { 47 | initCmd.Flags().BoolVarP(&autoUpdate, "update", "u", false, "Automatically update the new image") 48 | RootCmd.AddCommand(initCmd) 49 | } 50 | 51 | func doInit(manager *builder.Manager) { 52 | prof := manager.GetProfile() 53 | bk := builder.NewBackingImage(prof.Image) 54 | if bk.IsInstalled() { 55 | fmt.Printf("'%v' has already been initialised\n", profile) 56 | return 57 | } 58 | 59 | imgDir := builder.ImagesDir 60 | 61 | // Ensure directories exist 62 | if !builder.PathExists(imgDir) { 63 | if err := os.MkdirAll(imgDir, 00755); err != nil { 64 | log.WithFields(log.Fields{ 65 | "dir": imgDir, 66 | "error": err, 67 | }).Error("Failed to create images directory") 68 | os.Exit(1) 69 | } 70 | log.WithFields(log.Fields{ 71 | "dir": imgDir, 72 | }).Debug("Created images directory") 73 | } 74 | 75 | // Now ensure we actually have said image 76 | if !bk.IsFetched() { 77 | downloadImage(bk) 78 | } 79 | 80 | // Decompress the image 81 | log.WithFields(log.Fields{ 82 | "source": bk.ImagePathXZ, 83 | "target": bk.ImagePath, 84 | }).Debug("Decompressing backing image") 85 | 86 | if err := commands.ExecStdoutArgsDir(builder.ImagesDir, "unxz", []string{bk.ImagePathXZ}); err != nil { 87 | log.WithFields(log.Fields{ 88 | "source": bk.ImagePathXZ, 89 | "error": err, 90 | }).Error("Failed to decompress image") 91 | } 92 | 93 | log.WithFields(log.Fields{ 94 | "profile": profile, 95 | }).Info("Profile successfully initialised") 96 | } 97 | 98 | // Downloads an image using net/http. 99 | func downloadImage(bk *builder.BackingImage) (err error) { 100 | file, err := os.Create(bk.ImagePathXZ) 101 | if err != nil { 102 | log.WithFields(log.Fields{ 103 | "path": bk.ImagePathXZ, 104 | "error": err, 105 | }).Error("Failed to create file") 106 | return err 107 | } 108 | 109 | defer func() { 110 | if err != nil { 111 | os.Remove(bk.ImagePathXZ) 112 | } 113 | }() 114 | 115 | defer file.Close() 116 | 117 | resp, err := http.Get(bk.ImageURI) 118 | if err != nil { 119 | log.WithFields(log.Fields{ 120 | "uri": bk.ImageURI, 121 | "error": err, 122 | }).Error("Failed to fetch image") 123 | return err 124 | } 125 | 126 | defer resp.Body.Close() 127 | 128 | bar := pb.New64(resp.ContentLength).SetUnits(pb.U_BYTES) 129 | reader := bar.NewProxyReader(resp.Body) 130 | bar.ShowSpeed = true 131 | bar.Start() 132 | defer bar.Finish() 133 | 134 | bytesRemaining := resp.ContentLength 135 | done := false 136 | buf := make([]byte, 32*1024) 137 | for !done { 138 | bytesRead, err := reader.Read(buf) 139 | if err == io.EOF { 140 | done = true 141 | } else if err != nil { 142 | log.WithFields(log.Fields{ 143 | "uri": bk.ImageURI, 144 | "error": err, 145 | }).Error("Failed to fetch image") 146 | return err 147 | } 148 | 149 | _, err = file.Write(buf[:bytesRead]) 150 | if err != nil { 151 | log.WithFields(log.Fields{ 152 | "uri": bk.ImagePathXZ, 153 | "error": err, 154 | }).Error("Failed to write to file") 155 | return err 156 | } 157 | 158 | bytesRemaining -= int64(bytesRead) 159 | } 160 | 161 | return nil 162 | } 163 | 164 | // doUpdate will perform an update to the image after the initial init stage 165 | func doUpdate(manager *builder.Manager) { 166 | if err := manager.Update(); err != nil { 167 | os.Exit(1) 168 | } 169 | } 170 | 171 | func initProfile(cmd *cobra.Command, args []string) { 172 | if len(args) == 1 { 173 | profile = strings.TrimSpace(args[0]) 174 | } 175 | 176 | if CLIDebug { 177 | log.SetLevel(log.DebugLevel) 178 | } 179 | log.StandardLogger().Formatter.(*log.TextFormatter).DisableColors = builder.DisableColors 180 | 181 | if os.Geteuid() != 0 { 182 | fmt.Fprintf(os.Stderr, "You must be root to run init profiles\n") 183 | os.Exit(1) 184 | } 185 | 186 | // Now we'll update the newly initialised image 187 | manager, err := builder.NewManager() 188 | if err != nil { 189 | return 190 | } 191 | 192 | // Safety first.. 193 | if err = manager.SetProfile(profile); err != nil { 194 | return 195 | } 196 | 197 | doInit(manager) 198 | 199 | if autoUpdate { 200 | doUpdate(manager) 201 | } 202 | } 203 | -------------------------------------------------------------------------------- /src/solbuild/cmd/root.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2016-2017 Ikey Doherty 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package cmd 18 | 19 | import ( 20 | "builder" 21 | "github.com/spf13/cobra" 22 | "os" 23 | ) 24 | 25 | // Shared between most of the subcommands 26 | var profile string 27 | 28 | // CLIDebug determines whether to enable debug level logs or not. 29 | var CLIDebug bool 30 | 31 | // RootCmd is the main entry point into solbuild 32 | var RootCmd = &cobra.Command{ 33 | Use: "solbuild", 34 | Short: "solbuild is the Solus package builder", 35 | } 36 | 37 | func init() { 38 | RootCmd.PersistentFlags().StringVarP(&profile, "profile", "p", "", "Build profile to use") 39 | RootCmd.PersistentFlags().BoolVarP(&CLIDebug, "debug", "d", false, "Enable debug messages") 40 | RootCmd.PersistentFlags().BoolVarP(&builder.DisableColors, "no-color", "n", false, "Disable color output") 41 | } 42 | 43 | // FindLikelyArg will look in the current directory to see if common path names exist, 44 | // for when it is acceptable to omit a filename. 45 | func FindLikelyArg() string { 46 | lookPaths := []string{ 47 | "package.yml", 48 | "pspec.xml", 49 | } 50 | for _, p := range lookPaths { 51 | if st, err := os.Stat(p); err == nil { 52 | if st != nil { 53 | return p 54 | } 55 | } 56 | } 57 | return "" 58 | } 59 | -------------------------------------------------------------------------------- /src/solbuild/cmd/update.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2016-2017 Ikey Doherty 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package cmd 18 | 19 | import ( 20 | "builder" 21 | "fmt" 22 | log "github.com/Sirupsen/logrus" 23 | "github.com/spf13/cobra" 24 | "os" 25 | "strings" 26 | ) 27 | 28 | var updateCmd = &cobra.Command{ 29 | Use: "update", 30 | Short: "update a solbuild profile", 31 | Long: `Update the base image of the specified solbuild profile, helping to 32 | minimize the build times in future updates with this profile.`, 33 | Aliases: []string{"up"}, 34 | Run: updateProfile, 35 | } 36 | 37 | func init() { 38 | RootCmd.AddCommand(updateCmd) 39 | } 40 | 41 | func updateProfile(cmd *cobra.Command, args []string) { 42 | if len(args) == 1 { 43 | profile = strings.TrimSpace(args[0]) 44 | } 45 | 46 | if CLIDebug { 47 | log.SetLevel(log.DebugLevel) 48 | } 49 | log.StandardLogger().Formatter.(*log.TextFormatter).DisableColors = builder.DisableColors 50 | 51 | if os.Geteuid() != 0 { 52 | fmt.Fprintf(os.Stderr, "You must be root to run init profiles\n") 53 | os.Exit(1) 54 | } 55 | 56 | // Initialise the build manager 57 | manager, err := builder.NewManager() 58 | if err != nil { 59 | return 60 | } 61 | // Safety first.. 62 | if err = manager.SetProfile(profile); err != nil { 63 | if err == builder.ErrProfileNotInstalled { 64 | fmt.Fprintf(os.Stderr, "%v: Did you forget to init?\n", err) 65 | } 66 | return 67 | } 68 | 69 | if err := manager.Update(); err != nil { 70 | if err == builder.ErrProfileNotInstalled { 71 | fmt.Fprintf(os.Stderr, "%v: Did you forget to init?\n", err) 72 | } 73 | os.Exit(1) 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/solbuild/cmd/version.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2016-2017 Ikey Doherty 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package cmd 18 | 19 | import ( 20 | "fmt" 21 | "github.com/spf13/cobra" 22 | ) 23 | 24 | const ( 25 | // SolbuildVersion is the current public version of solbuild 26 | SolbuildVersion = "1.4.2" 27 | ) 28 | 29 | var versionCmd = &cobra.Command{ 30 | Use: "version", 31 | Short: "show version", 32 | Long: "Print the solbuild version and exit", 33 | Run: printVersion, 34 | } 35 | 36 | func init() { 37 | RootCmd.AddCommand(versionCmd) 38 | } 39 | 40 | func printVersion(cmd *cobra.Command, args []string) { 41 | fmt.Printf("solbuild version %v\n\nCopyright © 2016-2017 Solus Project\n", SolbuildVersion) 42 | fmt.Printf("Licensed under the Apache License, Version 2.0\n") 43 | } 44 | -------------------------------------------------------------------------------- /src/solbuild/main.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2016-2017 Ikey Doherty 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package main 18 | 19 | import ( 20 | _ "builder" 21 | log "github.com/Sirupsen/logrus" 22 | "os" 23 | "solbuild/cmd" 24 | ) 25 | 26 | // Set up the main logger formatting used in USpin 27 | func init() { 28 | form := &log.TextFormatter{} 29 | form.FullTimestamp = true 30 | form.TimestampFormat = "15:04:05" 31 | log.SetFormatter(form) 32 | } 33 | 34 | func main() { 35 | if err := cmd.RootCmd.Execute(); err != nil { 36 | os.Exit(1) 37 | } 38 | } 39 | --------------------------------------------------------------------------------