├── .gitignore
├── Kyuafile
├── AUTHORS
├── extra.mk.conf
├── CONTRIBUTORS
├── .travis.yml
├── sandbox.conf.in
├── default.conf.in
├── COPYING
├── CONTRIBUTING.md
├── sandbox.conf.pre
├── NEWS
├── README.md
├── admin
    ├── clean-all.sh
    ├── travis-build.sh
    ├── travis-install-deps.sh
    ├── bootstrap.sh
    └── make-macos-pkg.sh
├── pkgsrc.subr
├── pkg_comp4cron.8
├── configure.ac
├── pkgsrc_test.sh
├── sandbox.conf.post
├── pkg_comp4cron.sh
├── git.subr
├── pkg_comp4cron_test.sh
├── Makefile.am
├── INSTALL.md
├── pkg_comp.conf.5
├── git_test.sh
├── pkg_comp.8.in
├── pkg_comp_test.sh
├── pkg_comp.sh
└── pkg_comp_inttest.sh


/.gitignore:
--------------------------------------------------------------------------------
 1 | *_inttest
 2 | *_test
 3 | Makefile
 4 | Makefile.in
 5 | aclocal.m4
 6 | admin/install-sh
 7 | admin/missing
 8 | autom4te.cache
 9 | config.log
10 | config.status
11 | configure
12 | default.conf
13 | pkg_comp
14 | pkg_comp.8
15 | pkg_comp4cron
16 | sandbox.conf
17 | 


--------------------------------------------------------------------------------
/Kyuafile:
--------------------------------------------------------------------------------
 1 | syntax(2)
 2 | 
 3 | test_suite("pkg_comp")
 4 | 
 5 | plain_test_program{name="git_test", required_programs="git"}
 6 | plain_test_program{name="pkgsrc_test"}
 7 | atf_test_program{name="pkg_comp_inttest", is_exclusive=true}
 8 | atf_test_program{name="pkg_comp_test"}
 9 | plain_test_program{name="pkg_comp4cron_test"}
10 | 


--------------------------------------------------------------------------------
/AUTHORS:
--------------------------------------------------------------------------------
 1 | # This is the official list of pkg_comp authors for copyright purposes.
 2 | #
 3 | # This file is distinct from the CONTRIBUTORS files; see the latter for
 4 | # an explanation.
 5 | #
 6 | # Names are sorted alphabetically and should be added to this file as:
 7 | #
 8 | #     * Name <email address>
 9 | #     * Organization <optional email address>
10 | 
11 | * Google Inc.
12 | 


--------------------------------------------------------------------------------
/extra.mk.conf:
--------------------------------------------------------------------------------
 1 | # Configuration file to extend the mk.conf(5) created by pkg_comp(8).
 2 | #
 3 | # This file is included by the mk.conf file created within the sandbox by
 4 | # pkg_comp(8).  Use this to customize the build of your packages, but do NOT
 5 | # configure the file layout here (such as by setting LOCALBASE).
 6 | #
 7 | # The contents of this file are automatically guarded by BSD_PKG_MK so you don't
 8 | # need to worry about this here.
 9 | 
10 | #ACCEPTABLE_LICENSES=
11 | #PKG_DEFAULT_OPTIONS=
12 | 


--------------------------------------------------------------------------------
/CONTRIBUTORS:
--------------------------------------------------------------------------------
 1 | # This is the list of people who have agreed to one of the CLAs and can
 2 | # contribute patches to the pkg_comp project.
 3 | #
 4 | # The AUTHORS file lists the copyright holders; this file lists people.
 5 | # For example: Google employees are listed here but not in AUTHORS
 6 | # because Google holds the copyright.
 7 | #
 8 | # See the CONTRIBUTING.md file for more details on the CLA.
 9 | #
10 | # Names are sorted by last name and should be added as:
11 | #
12 | #     * Name <email address>
13 | 
14 | * Julio Merino <jmmv@google.com>
15 | 


--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
 1 | dist: trusty
 2 | sudo: required
 3 | 
 4 | before_install:
 5 |     - ./admin/travis-install-deps.sh
 6 | 
 7 | env:
 8 |     - AS_ROOT=no SHELL_NAME=bash
 9 |     - AS_ROOT=no SHELL_NAME=dash
10 |     - AS_ROOT=no SHELL_NAME=pdksh
11 |     - AS_ROOT=no SHELL_NAME=zsh
12 |     - AS_ROOT=yes INTTEST_SHARD=0 INTTEST_SHARDS=4 SHELL_NAME=dash
13 |     - AS_ROOT=yes INTTEST_SHARD=1 INTTEST_SHARDS=4 SHELL_NAME=dash
14 |     - AS_ROOT=yes INTTEST_SHARD=2 INTTEST_SHARDS=4 SHELL_NAME=dash
15 |     - AS_ROOT=yes INTTEST_SHARD=3 INTTEST_SHARDS=4 SHELL_NAME=dash
16 | 
17 | script:
18 |     - travis_wait 60 ./admin/travis-build.sh
19 | 


--------------------------------------------------------------------------------
/sandbox.conf.in:
--------------------------------------------------------------------------------
 1 | # Configuration file for sandboxctl(8) to use with pkg_comp(8).
 2 | #
 3 | # The default configuration provided by this file has reasonable chances of
 4 | # working out of the box on a system supported by sandboxctl(8), but the user
 5 | # should review these contents regardless.
 6 | #
 7 | # See sandboxctl.conf(5) for details.
 8 | 
 9 | # Inherit the default configuration for sandboxctl, which should be functional
10 | # for the current system even if the user has not yet tuned it.
11 | #
12 | # RECOMMENDED: Remove this chunk of configuration, explicitly set SANDBOX_TYPE
13 | # to match your system, and define any required type-specific settings.
14 | if [ -e '__PKG_COMP_BASESYSCONFDIR__/sandboxctl/default.conf' ]; then
15 |     . '__PKG_COMP_BASESYSCONFDIR__/sandboxctl/default.conf'
16 | else
17 |     # No valid system-wide sandboxctl configuration.  At the very least
18 |     # you must set the sandbox type, but you will also probably need to
19 |     # configure type-specific settings.
20 |     #SANDBOX_TYPE=netbsd-native
21 |     :
22 | fi
23 | 
24 | # Location to use for the creation of the sandbox.
25 | SANDBOX_ROOT=/var/tmp/pkg_comp-sandbox
26 | 


--------------------------------------------------------------------------------
/default.conf.in:
--------------------------------------------------------------------------------
 1 | # Configuration file for pkg_comp(8); see pkg_comp.conf(5) for details.
 2 | #
 3 | # This configuration file provides a sample setup to build a collection of
 4 | # packages for the local machine.
 5 | 
 6 | # Additional configuration files to support this setup.
 7 | EXTRA_MKCONF="__PKG_COMP_ETCDIR__/extra.mk.conf"
 8 | SANDBOX_CONFFILE="__PKG_COMP_ETCDIR__/sandbox.conf"
 9 | 
10 | # Remote VCS configuration.
11 | #FETCH_VCS=cvs
12 | #CVS_ROOT=:ext:anoncvs@anoncvs.NetBSD.org:/cvsroot
13 | #CVS_TAG=pkgsrc-2017Q1
14 | #GIT_URL=https://github.com/NetBSD/pkgsrc.git
15 | #GIT_BRANCH=trunk
16 | 
17 | # Host file layout.  These directories point to trees managed outside of
18 | # the sandbox and are exposed within the sandbox via null mounts.
19 | PKGSRCDIR="/usr/pkgsrc"
20 | DISTDIR="${PKGSRCDIR}/distfiles"
21 | PACKAGES="${PKGSRCDIR}/packages"
22 | PBULK_LOG="${PACKAGES}/log"
23 | PBULK_PACKAGES="${PACKAGES}/pbulk"
24 | 
25 | # Target file layout.  These are the directories that the built binary
26 | # packages will use when installed.
27 | #LOCALBASE=/usr/pkg
28 | #PKG_DBDIR=/usr/pkg/libdata/pkgdb
29 | #SYSCONFDIR=/etc
30 | #VARBASE=/var
31 | 
32 | # Enable interactive pkgsrc development via sandbox-shell.
33 | #PKG_DEVELOPER=yes
34 | 
35 | # List of packages to build during automatic execution.
36 | #AUTO_PACKAGES="bash emacs kyua pkg_comp tmux"
37 | 


--------------------------------------------------------------------------------
/COPYING:
--------------------------------------------------------------------------------
 1 | Copyright 2013 Google Inc.
 2 | All rights reserved.
 3 | 
 4 | Redistribution and use in source and binary forms, with or without
 5 | modification, are permitted provided that the following conditions are
 6 | met:
 7 | 
 8 | * Redistributions of source code must retain the above copyright
 9 |   notice, this list of conditions and the following disclaimer.
10 | * Redistributions in binary form must reproduce the above copyright
11 |   notice, this list of conditions and the following disclaimer in the
12 |   documentation and/or other materials provided with the distribution.
13 | * Neither the name of Google Inc. nor the names of its contributors
14 |   may be used to endorse or promote products derived from this software
15 |   without specific prior written permission.
16 | 
17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 | 


--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
 1 | # Contributing
 2 | 
 3 | Want to contribute?  Great!  First, read this page (including the small print
 4 | at the end).
 5 | 
 6 | ## Before you contribute
 7 | 
 8 | Before we can use your code, you must sign the [Google Individual Contributor
 9 | License Agreement](https://cla.developers.google.com/about/google-individual)
10 | (CLA), which you can do online.
11 | 
12 | The CLA is necessary mainly because you own the copyright to your changes, even
13 | after your contribution becomes part of our codebase, so we need your
14 | permission to use and distribute your code.  We also need to be sure of various
15 | other things: for instance that you'll tell us if you know that your code
16 | infringes on other people's patents.  You don't have to sign the CLA until
17 | after you've submitted your code for review and a member has approved it, but
18 | you must do it before we can put your code into our codebase.
19 | 
20 | Before you start working on a larger contribution, you should get in touch with
21 | us first through the issue tracker with your idea so that we can help out and
22 | possibly guide you.  Coordinating up front makes it much easier to avoid
23 | frustration later on.
24 | 
25 | ## Code reviews
26 | 
27 | All submissions, including submissions by project members, require review.
28 | We use Github pull requests for this purpose.
29 | 
30 | ## The small print
31 | 
32 | Contributions made by corporations are covered by a different agreement than
33 | the one above, the [Software Grant and Corporate Contributor License
34 | Agreement](https://cla.developers.google.com/about/google-corporate).
35 | 


--------------------------------------------------------------------------------
/sandbox.conf.pre:
--------------------------------------------------------------------------------
 1 | # Copyright 2017 Google Inc.
 2 | # All rights reserved.
 3 | #
 4 | # Redistribution and use in source and binary forms, with or without
 5 | # modification, are permitted provided that the following conditions are
 6 | # met:
 7 | #
 8 | # * Redistributions of source code must retain the above copyright
 9 | #   notice, this list of conditions and the following disclaimer.
10 | # * Redistributions in binary form must reproduce the above copyright
11 | #   notice, this list of conditions and the following disclaimer in the
12 | #   documentation and/or other materials provided with the distribution.
13 | # * Neither the name of Google Inc. nor the names of its contributors
14 | #   may be used to endorse or promote products derived from this software
15 | #   without specific prior written permission.
16 | #
17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 | 
29 | # \file sandbox.conf.pre
30 | # Configuration prepended to the sandbox.conf(5) file generated by pkg_comp(8).
31 | 
32 | 
33 | # User-settable sandbox post-creation hook.
34 | pkg_comp_post_create_hook() {
35 |     true
36 | }
37 | 
38 | 
39 | # User-settable sandbox post-mount hook.
40 | pkg_comp_post_mount_hook() {
41 |     true
42 | }
43 | 


--------------------------------------------------------------------------------
/NEWS:
--------------------------------------------------------------------------------
 1 | Changes in version 2.1
 2 | ======================
 3 | 
 4 | **Released on 2019-01-14.**
 5 | 
 6 | * Issue 3: Added a new `PBULK_LOG` setting to point at a location that
 7 |   holds all build logs, and made the `build` command print out details
 8 |   about such logs on success and failure.
 9 | 
10 | * Issue 4: Fixed the installable bootstrap kit generated during a build
11 |   to not leak pkg_comp-internal paths.
12 | 
13 | * Issue 10: Fixed generation of pkg_summary so that it includes all
14 |   packages and not only those that were explicitly given to the `auto`
15 |   or `build` commands after a clean rebuild of the sandbox.
16 | 
17 | * PR 20: Fixed the default Git repository from jsonn's to NetBSD's as
18 |   it has moved upstream.
19 | 
20 | * Added a new `PKG_DEVELOPER` setting to enable pkgsrc development
21 |   from within the sandbox.
22 | 
23 | 
24 | Changes in version 2.0
25 | ======================
26 | 
27 | **Released on 2017-02-17.**
28 | 
29 | This is the first release of the pkg_comp project as a standalone
30 | package.  This new release shares no code with previous versions and is
31 | not compatible with them.
32 | 
33 | The following are the major differences between pkg_comp 2.0 and all
34 | previous releases, which incidentally are the reasons that triggered
35 | this rewrite:
36 | 
37 | * Support for multiple platforms.
38 | 
39 | * Use of pbulk to (re)build packages within the sandbox.  This results
40 |   in more reliable incremental builds after pkgsrc updates.
41 | 
42 | * Support for bootstrap, generating binary kits as part of the builds.
43 | 
44 | * Better scriptability to allow running from cron(8) trivially.
45 | 
46 | 
47 | 1.x versions
48 | ============
49 | 
50 | pkg_comp 1.x existed in pkgsrc from 2002-09-06 under the
51 | pkgtools/pkg_comp subdirectory and was later moved to pkgtools/pkg_comp1
52 | on 2017-02-12.
53 | 
54 | The history of those releases can be found in the pkgsrc repository,
55 | but as the codebase was so different that the new code, I did not
56 | bother to port the notes to this file.
57 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Welcome to pkg_comp, the pkgsrc compiler!
 2 | 
 3 | [pkgsrc](http://pkgsrc.org) is a framework for building third-party
 4 | software on a multitude of platforms, including (but not limited to):
 5 | FreeBSD, Linux, macOS, and NetBSD.
 6 | 
 7 | **pkg_comp is a tool for building and maintaining a repository of binary
 8 | pkgsrc packages.** pkg_comp does this by building the packages in a clean
 9 | and self-contained environment, hiding all complexity behind a simple
10 | command and configuration file.  pkg_comp makes it even possible to
11 | automate the builds of packages from cron so that you always have an
12 | up-to-date local repository.
13 | 
14 | pkg_comp orchestrates VCS tools such as CVS and Git to **fetch the initial
15 | pkgsrc tree** and to keep it up to date; the **creation of a fresh
16 | sandbox** to build packages in; the **pkgsrc bootstrapping process**; and
17 | the **use of the pbulk infrastructure** to build the desired set of
18 | packages in an optimal manner.
19 | 
20 | With pkg_comp, you can quickly and effortlessly get third-party software up
21 | and running, and you can target both your own machine or other machines
22 | with different configurations.  As you build the packages from source,
23 | pkgsrc puts you in control by letting you tune the software to your needs.
24 | 
25 | pkg_comp is licensed under a **[liberal BSD 3-clause license](COPYING)**.
26 | This is not an official Google product.
27 | 
28 | ## Download and installation
29 | 
30 | The latest version is **[pkg_comp 2.1](../../releases/tag/pkg_comp-2.1)**
31 | and was released on January 14th, 2019.
32 | 
33 | **Read the [installation instructions](INSTALL.md)** for details on which
34 | file to download and how to get started.
35 | 
36 | **Read the [release notes](NEWS)** for information about the changes in
37 | this and all previous releases.
38 | 
39 | ## Contributing
40 | 
41 | Want to contribute?  Great!  But please first read the guidelines provided
42 | in [CONTRIBUTING.md](CONTRIBUTING.md).
43 | 
44 | If you are curious about who made this project possible, you can check out
45 | the [list of copyright holders](AUTHORS) and the [list of
46 | individuals](CONTRIBUTORS).
47 | 
48 | ## Support
49 | 
50 | Please use the [bug tracker](https://github.com/jmmv/pkg_comp/issues) for
51 | any support inquiries.
52 | 
53 | *Homepage:* https://github.com/jmmv/pkg_comp/
54 | 


--------------------------------------------------------------------------------
/admin/clean-all.sh:
--------------------------------------------------------------------------------
 1 | #! /bin/sh
 2 | # Copyright 2010 Google Inc.
 3 | # All rights reserved.
 4 | #
 5 | # Redistribution and use in source and binary forms, with or without
 6 | # modification, are permitted provided that the following conditions are
 7 | # met:
 8 | #
 9 | # * Redistributions of source code must retain the above copyright
10 | #   notice, this list of conditions and the following disclaimer.
11 | # * Redistributions in binary form must reproduce the above copyright
12 | #   notice, this list of conditions and the following disclaimer in the
13 | #   documentation and/or other materials provided with the distribution.
14 | # * Neither the name of Google Inc. nor the names of its contributors
15 | #   may be used to endorse or promote products derived from this software
16 | #   without specific prior written permission.
17 | #
18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 | 
30 | Prog_Name=${0##*/}
31 | 
32 | if [ ! -f ./pkg_comp.sh ]; then
33 |     echo "${Prog_Name}: must be run from the source top directory" 1>&2
34 |     exit 1
35 | fi
36 | 
37 | if [ ! -f configure ]; then
38 |     echo "${Prog_Name}: configure not found; nothing to clean?" 1>&2
39 |     exit 1
40 | fi
41 | 
42 | [ -f Makefile ] || ./configure
43 | make distclean
44 | 
45 | # Top-level directory.
46 | rm -f Makefile.in
47 | rm -f aclocal.m4
48 | rm -rf autom4te.cache
49 | rm -f configure
50 | rm -f pkg_comp-*.tar.gz
51 | 
52 | # admin directory.
53 | rm -f admin/install-sh
54 | rm -f admin/missing
55 | 
56 | # Files and directories spread all around the tree.
57 | find . -name '#*' | xargs rm -rf
58 | find . -name '*~' | xargs rm -rf
59 | 
60 | # Show remaining files.
61 | if [ -n "${GIT}" ]; then
62 |     echo ">>> untracked and ignored files"
63 |     "${GIT}" status --porcelain --ignored | grep -E '^(\?\?|!!)' || true
64 | fi
65 | 


--------------------------------------------------------------------------------
/admin/travis-build.sh:
--------------------------------------------------------------------------------
 1 | #! /bin/sh
 2 | # Copyright 2017 Google Inc.
 3 | # All rights reserved.
 4 | #
 5 | # Redistribution and use in source and binary forms, with or without
 6 | # modification, are permitted provided that the following conditions are
 7 | # met:
 8 | #
 9 | # * Redistributions of source code must retain the above copyright
10 | #   notice, this list of conditions and the following disclaimer.
11 | # * Redistributions in binary form must reproduce the above copyright
12 | #   notice, this list of conditions and the following disclaimer in the
13 | #   documentation and/or other materials provided with the distribution.
14 | # * Neither the name of Google Inc. nor the names of its contributors
15 | #   may be used to endorse or promote products derived from this software
16 | #   without specific prior written permission.
17 | #
18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 | 
30 | set -e -x
31 | 
32 | autoreconf -isv -I/usr/local/share/aclocal
33 | 
34 | ./configure
35 | 
36 | base="$(pwd)/base"
37 | 
38 | git clone https://github.com/jsonn/pkgsrc.git "${base}/pkgsrc"
39 | 
40 | cat >kyua.conf <<EOF
41 | syntax(2)
42 | unprivileged_user = 'travis'
43 | test_suites.pkg_comp.distdir = '${base}/distfiles'
44 | test_suites.pkg_comp.packages = '${base}/packages'
45 | test_suites.pkg_comp.pkgsrcdir = '${base}/pkgsrc'
46 | test_suites.pkg_comp.sandbox_conffile = '${base}/sandbox.conf'
47 | EOF
48 | 
49 | cat >"${base}/sandbox.conf" <<EOF
50 | SANDBOX_ROOT="${base}/sandbox"
51 | SANDBOX_TYPE=linux-native
52 | EOF
53 | 
54 | mkdir -p distfiles packages
55 | 
56 | f=
57 | f="${f} KYUA_FLAGS='-c $(pwd)/kyua.conf'"
58 | f="${f} PKG_CONFIG_PATH='/usr/local/lib/pkgconfig'"
59 | if [ "${AS_ROOT:-no}" = yes ]; then
60 |     sudo -H PATH="${PATH}" make distcheck DISTCHECK_CONFIGURE_FLAGS="${f}"
61 | else
62 |     make distcheck DISTCHECK_CONFIGURE_FLAGS="${f}"
63 | fi
64 | 


--------------------------------------------------------------------------------
/admin/travis-install-deps.sh:
--------------------------------------------------------------------------------
 1 | #! /bin/sh
 2 | # Copyright 2017 Google Inc.
 3 | # All rights reserved.
 4 | #
 5 | # Redistribution and use in source and binary forms, with or without
 6 | # modification, are permitted provided that the following conditions are
 7 | # met:
 8 | #
 9 | # * Redistributions of source code must retain the above copyright
10 | #   notice, this list of conditions and the following disclaimer.
11 | # * Redistributions in binary form must reproduce the above copyright
12 | #   notice, this list of conditions and the following disclaimer in the
13 | #   documentation and/or other materials provided with the distribution.
14 | # * Neither the name of Google Inc. nor the names of its contributors
15 | #   may be used to endorse or promote products derived from this software
16 | #   without specific prior written permission.
17 | #
18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 | 
30 | set -e -x
31 | 
32 | sudo apt-get update -qq
33 | sudo apt-get install -y cvs liblua5.2-0 libsqlite3-0 pkg-config ${SHELL_NAME}
34 | 
35 | install_from_head() {
36 |     local user="${1}"; shift
37 |     local repo="${1}"; shift
38 | 
39 |     git clone "https://github.com/${user}/${repo}"
40 | 
41 |     cd "${repo}"
42 |     autoreconf -isv -I/usr/local/share/aclocal
43 |     ./configure \
44 |         --without-atf \
45 |         PKG_CONFIG_PATH="/usr/local/lib/pkgconfig" \
46 |         "${@}"
47 |     make
48 |     sudo make install
49 |     cd -
50 | 
51 |     rm -rf "${repo}"
52 | }
53 | 
54 | install_kyua() {
55 |     local name="20170225-usr-local-kyua-ubuntu-14-04-amd64-clang.tar.gz"
56 |     wget "http://dl.bintray.com/jmmv/kyua/${name}" || return 1
57 |     sudo tar -xzvp -C / -f "${name}"
58 |     rm -f "${name}"
59 | }
60 | 
61 | install_kyua
62 | 
63 | shell_path="$(which ${SHELL_NAME})"
64 | if [ ${?} -ne 0 ]; then
65 |     echo "Failed to resolve path to shell ${SHELL_NAME}" 1>&2
66 |     exit 1
67 | fi
68 | install_from_head jmmv shtk SHTK_SHELL="${shell_path}"
69 | install_from_head jmmv sandboxctl
70 | 


--------------------------------------------------------------------------------
/pkgsrc.subr:
--------------------------------------------------------------------------------
 1 | # Copyright 2017 Google Inc.
 2 | # All rights reserved.
 3 | #
 4 | # Redistribution and use in source and binary forms, with or without
 5 | # modification, are permitted provided that the following conditions are
 6 | # met:
 7 | #
 8 | # * Redistributions of source code must retain the above copyright
 9 | #   notice, this list of conditions and the following disclaimer.
10 | # * Redistributions in binary form must reproduce the above copyright
11 | #   notice, this list of conditions and the following disclaimer in the
12 | #   documentation and/or other materials provided with the distribution.
13 | # * Neither the name of Google Inc. nor the names of its contributors
14 | #   may be used to endorse or promote products derived from this software
15 | #   without specific prior written permission.
16 | #
17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 | 
29 | # \file pkgsrc.subr
30 | # Utilities to interact with pkgsrc.
31 | 
32 | shtk_import cli
33 | 
34 | 
35 | # Validates package names and resolves those that don't specify categories.
36 | #
37 | # Given a list of words of the form "category/pkgname" or "pkgname", ensures
38 | # that those packages exist and prints the name of all valid packages.  For
39 | # words of the form "pkgname", the output will be the corresponding
40 | # "category/pkgname" for the entry.
41 | #
42 | # \param pkgsrcdir Path to the pkgsrc tree.
43 | # \param ... Package names to expand.
44 | #
45 | # \return 0 if all packages exist; 1 otherwise.  Any non-existent packages
46 | # are logged to stderr as a warning.
47 | pkgsrc_expand_packages() {
48 |     local pkgsrcdir="${1}"; shift
49 | 
50 |     local failed=no
51 |     for package in "${@}"; do
52 |         local candidate
53 |         case "${package}" in
54 |             */*) candidate="${package}" ;;
55 |             *) candidate="$(cd "${pkgsrcdir}" \
56 |                 && echo */"${package}/Makefile" | cut -d / -f -2)" ;;
57 |         esac
58 |         if [ ! -e "${pkgsrcdir}/${candidate}/Makefile" ]; then
59 |             shtk_cli_warning "Package ${package} does not exist"
60 |             failed=yes
61 |         else
62 |             echo "${candidate}"
63 |         fi
64 |     done
65 |     [ "${failed}" = no ]
66 | }
67 | 


--------------------------------------------------------------------------------
/pkg_comp4cron.8:
--------------------------------------------------------------------------------
 1 | .\" Copyright 2012 Google Inc.
 2 | .\" All rights reserved.
 3 | .\"
 4 | .\" Redistribution and use in source and binary forms, with or without
 5 | .\" modification, are permitted provided that the following conditions are
 6 | .\" met:
 7 | .\"
 8 | .\" * Redistributions of source code must retain the above copyright
 9 | .\"   notice, this list of conditions and the following disclaimer.
10 | .\" * Redistributions in binary form must reproduce the above copyright
11 | .\"   notice, this list of conditions and the following disclaimer in the
12 | .\"   documentation and/or other materials provided with the distribution.
13 | .\" * Neither the name of Google Inc. nor the names of its contributors
14 | .\"   may be used to endorse or promote products derived from this software
15 | .\"   without specific prior written permission.
16 | .\"
17 | .\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 | .\" "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 | .\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 | .\" A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 | .\" OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 | .\" SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 | .\" LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 | .\" DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 | .\" THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 | .\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 | .\" OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 | .Dd February 1, 2017
29 | .Dt PKG_COMP4CRON 1
30 | .Os
31 | .Sh NAME
32 | .Nm pkg_comp4cron
33 | .Nd performs fully automatic builds of pkgsrc packages from cron
34 | .Sh SYNOPSIS
35 | .Nm
36 | .Op Fl l Ar logdir
37 | .Op Fl r Ar recipient
38 | --
39 | .Ar pkg_comp_arguments
40 | .Sh DESCRIPTION
41 | .Nm
42 | is a wrapper over
43 | .Xr pkg_comp 8
44 | that allows plugging pkgsrc builds into
45 | .Xr cron 8
46 | with minimal effort.
47 | .Pp
48 | .Nm
49 | executes
50 | .Xr pkg_comp 8
51 | with the given arguments after the
52 | .Sq --
53 | marker, stores the output of the build in a log file and, if the exit code of
54 | the tool indicated an error, proceeds to send an error report to a given
55 | recipient.
56 | .Pp
57 | Unless
58 | .Nm
59 | detects an error in the usage of this wrapper script, the tool does not print
60 | any output to the console and does not return any error codes.
61 | All errors are reported by means of email.
62 | .Pp
63 | The following options are supported:
64 | .Bl -tag -width XXXX
65 | .It Fl l Ar logdir
66 | Path to the directory that will keep the logs of all the calls to
67 | .Xr pkg_comp 1 .
68 | .Pp
69 | Default:
70 | .Pa ~/pkg_comp/log .
71 | .It Fl r Ar recipient
72 | Email address of the recipient of the failure messages.
73 | .Pp
74 | Default: the current username, without a domain part.
75 | .El
76 | .Sh SEE ALSO
77 | .Xr crontab 1 ,
78 | .Xr pkg_comp 1 ,
79 | .Xr crontab 5
80 | .Sh AUTHORS
81 | The
82 | .Nm
83 | utility was developed by
84 | .An Julio Merino
85 | .Aq jmmv@google.com .
86 | 


--------------------------------------------------------------------------------
/configure.ac:
--------------------------------------------------------------------------------
 1 | dnl Copyright 2013 Google Inc.
 2 | dnl All rights reserved.
 3 | dnl
 4 | dnl Redistribution and use in source and binary forms, with or without
 5 | dnl modification, are permitted provided that the following conditions are
 6 | dnl met:
 7 | dnl
 8 | dnl * Redistributions of source code must retain the above copyright
 9 | dnl   notice, this list of conditions and the following disclaimer.
10 | dnl * Redistributions in binary form must reproduce the above copyright
11 | dnl   notice, this list of conditions and the following disclaimer in the
12 | dnl   documentation and/or other materials provided with the distribution.
13 | dnl * Neither the name of Google Inc. nor the names of its contributors
14 | dnl   may be used to endorse or promote products derived from this software
15 | dnl   without specific prior written permission.
16 | dnl
17 | dnl THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 | dnl "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 | dnl LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 | dnl A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 | dnl OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 | dnl SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 | dnl LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 | dnl DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 | dnl THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 | dnl (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 | dnl OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 | 
29 | AC_INIT([pkg_comp], [2.1],
30 |         [https://github.com/jmmv/pkg_comp/issues/], [pkg_comp], [])
31 | AC_PREREQ([2.65])
32 | 
33 | 
34 | AC_COPYRIGHT([Copyright 2013 Google Inc.])
35 | AC_CONFIG_AUX_DIR([admin])
36 | AC_CONFIG_FILES([Makefile])
37 | AC_CONFIG_SRCDIR([pkg_comp.sh])
38 | 
39 | 
40 | AM_INIT_AUTOMAKE([1.9 check-news foreign subdir-objects -Wall])
41 | 
42 | 
43 | AC_PATH_PROG([SANDBOXCTL], [sandboxctl], [not-found])
44 | if test "${SANDBOXCTL}" = not-found; then
45 |     AC_MSG_ERROR([Cannot find sandboxctl])
46 | fi
47 | 
48 | 
49 | AC_ARG_VAR([PKG_COMP_CONFSUBDIR],
50 |            [Subdirectory of sysconfdir under which to look for files])
51 | if test x"${PKG_COMP_CONFSUBDIR-unset}" = x"unset"; then
52 |     PKG_COMP_CONFSUBDIR=pkg_comp
53 | else
54 |     case "${PKG_COMP_CONFSUBDIR}" in
55 |     /*)
56 |         AC_MSG_ERROR([PKG_COMP_CONFSUBDIR must hold a relative path])
57 |         ;;
58 |     *)
59 |         ;;
60 |     esac
61 | fi
62 | if test x"${PKG_COMP_CONFSUBDIR}" = x""; then
63 |     AC_SUBST(pkg_comp_confdir, \${sysconfdir})
64 | else
65 |     AC_SUBST(pkg_comp_confdir, \${sysconfdir}/\${PKG_COMP_CONFSUBDIR})
66 | fi
67 | 
68 | 
69 | m4_ifndef([PKG_CHECK_MODULES],
70 |     [m4_fatal([Cannot find pkg.m4; see the INSTALL document for help])])
71 | 
72 | m4_ifndef([SHTK_CHECK],
73 |     [m4_fatal([Cannot find shtk.m4; see the INSTALL document for help])])
74 | SHTK_CHECK([>= 1.7])
75 | 
76 | m4_ifndef([ATF_CHECK_SH],
77 |     [m4_fatal([Cannot find atf-sh.m4; see the INSTALL document for help])])
78 | ATF_CHECK_SH([>= 0.17])
79 | m4_ifndef([ATF_ARG_WITH],
80 |     [m4_fatal([Cannot find atf-common.m4; see the INSTALL document for help])])
81 | ATF_ARG_WITH
82 | 
83 | 
84 | AC_PATH_PROG([KYUA], [kyua])
85 | AM_CONDITIONAL([HAVE_KYUA], [test -n "${KYUA}"])
86 | AC_ARG_VAR([KYUA_FLAGS],
87 |            [Additional Kyua flags to use at 'make (|dist|install)check' time])
88 | 
89 | 
90 | AC_SUBST(pkgtestsdir, \${testsdir}/pkg_comp)
91 | AC_SUBST(testsdir, \${exec_prefix}/tests)
92 | 
93 | 
94 | AC_OUTPUT
95 | 


--------------------------------------------------------------------------------
/pkgsrc_test.sh:
--------------------------------------------------------------------------------
  1 | # Copyright 2017 Google Inc.
  2 | # All rights reserved.
  3 | #
  4 | # Redistribution and use in source and binary forms, with or without
  5 | # modification, are permitted provided that the following conditions are
  6 | # met:
  7 | #
  8 | # * Redistributions of source code must retain the above copyright
  9 | #   notice, this list of conditions and the following disclaimer.
 10 | # * Redistributions in binary form must reproduce the above copyright
 11 | #   notice, this list of conditions and the following disclaimer in the
 12 | #   documentation and/or other materials provided with the distribution.
 13 | # * Neither the name of Google Inc. nor the names of its contributors
 14 | #   may be used to endorse or promote products derived from this software
 15 | #   without specific prior written permission.
 16 | #
 17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 18 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 19 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 20 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 21 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 22 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 23 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 24 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 25 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 26 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 28 | 
 29 | shtk_import unittest
 30 | : ${PKG_COMP_SHTK_MODULESDIR:="__PKG_COMP_SHTK_MODULESDIR__"}
 31 | SHTK_MODULESPATH="${PKG_COMP_SHTK_MODULESDIR}" shtk_import pkgsrc
 32 | 
 33 | 
 34 | shtk_unittest_add_fixture expand_packages
 35 | expand_packages_fixture() {
 36 |     setup() {
 37 |         for pkg in first/pkg-a second/pkg-b third/pkg-c fourth/pkg-d; do
 38 |             mkdir -p "pkgsrc/${pkg}"
 39 |             touch "pkgsrc/${pkg}/Makefile"
 40 |         done
 41 |     }
 42 | 
 43 | 
 44 |     teardown() {
 45 |         rm -rf pkgsrc
 46 |     }
 47 | 
 48 | 
 49 |     shtk_unittest_add_test all_exist_with_categories
 50 |     all_exist_with_categories_test() {
 51 |         assert_command \
 52 |             -o not-match:pkg-a \
 53 |             -o match:second/pkg-b \
 54 |             -o not-match:pkg-c \
 55 |             -o match:fourth/pkg-d \
 56 |             pkgsrc_expand_packages pkgsrc second/pkg-b fourth/pkg-d
 57 |     }
 58 | 
 59 | 
 60 |     shtk_unittest_add_test all_exist_without_categories
 61 |     all_exist_without_categories_test() {
 62 |         assert_command \
 63 |             -o not-match:pkg-a \
 64 |             -o match:second/pkg-b \
 65 |             -o match:third/pkg-c \
 66 |             -o not-match:pkg-d \
 67 |             pkgsrc_expand_packages pkgsrc pkg-b pkg-c
 68 |     }
 69 | 
 70 | 
 71 |     shtk_unittest_add_test some_missing
 72 |     some_missing_test() {
 73 |         assert_command -s exit:1 \
 74 |             -o not-match:first \
 75 |             -o match:second/pkg-b \
 76 |             -o not-match:second/pkg-a \
 77 |             -o not-match:pkg-c \
 78 |             -o match:fourth/pkg-d \
 79 |             -o not-match:pkg-e \
 80 |             -e match:"W: Package first does not exist" \
 81 |             -e match:"W: Package second/pkg-a does not exist" \
 82 |             -e match:"W: Package pkg-e does not exist" \
 83 |             pkgsrc_expand_packages pkgsrc \
 84 |             first pkg-b second/pkg-a fourth/pkg-d pkg-e
 85 |     }
 86 | 
 87 | 
 88 |     shtk_unittest_add_test conflicting_names_in_distfiles
 89 |     conflicting_names_in_distfiles_test() {
 90 |         mkdir -p pkgsrc/distfiles/pkg-z
 91 |         mkdir -p pkgsrc/zzz/pkg-z
 92 |         touch pkgsrc/zzz/pkg-z/Makefile
 93 | 
 94 |         assert_command \
 95 |             -o match:zzz/pkg-z \
 96 |             -o not-match:distfiles \
 97 |             pkgsrc_expand_packages pkgsrc pkg-z
 98 |     }
 99 | }
100 | 


--------------------------------------------------------------------------------
/sandbox.conf.post:
--------------------------------------------------------------------------------
 1 | # Copyright 2013 Google Inc.
 2 | # All rights reserved.
 3 | #
 4 | # Redistribution and use in source and binary forms, with or without
 5 | # modification, are permitted provided that the following conditions are
 6 | # met:
 7 | #
 8 | # * Redistributions of source code must retain the above copyright
 9 | #   notice, this list of conditions and the following disclaimer.
10 | # * Redistributions in binary form must reproduce the above copyright
11 | #   notice, this list of conditions and the following disclaimer in the
12 | #   documentation and/or other materials provided with the distribution.
13 | # * Neither the name of Google Inc. nor the names of its contributors
14 | #   may be used to endorse or promote products derived from this software
15 | #   without specific prior written permission.
16 | #
17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 | 
29 | # \file sandbox-hooks.conf
30 | # Configuration appended to the sandbox.conf(5) file generated by pkg_comp(8).
31 | 
32 | shtk_import bool
33 | shtk_import cli
34 | 
35 | 
36 | # Verifies that the configuration variables that we expect are set.
37 | #
38 | # \post Terminates execution if any of the key configuration variables cannot be
39 | # accessed.
40 | _pkg_comp_validate_variables() {
41 |     for var in DISTDIR PACKAGES PKGSRCDIR PKG_DEVELOPER SANDBOX_ROOT; do
42 |         local value
43 |         eval "value=\"\$${var}\""
44 |         [ -n "${value}" ] || shtk_cli_error "Aborting; ${var} is not set"
45 |     done
46 | }
47 | 
48 | 
49 | # Creates the pkg_comp-specific file system hierarchy within the sandbox.
50 | post_create_hook() {
51 |     _pkg_comp_validate_variables
52 | 
53 |     # It is all too easy to forget to create the directories outside of the
54 |     # sandbox, which would then cause a rather obscure "mount" failure.
55 |     # Therefore, create them here for simplicity.  In principle this is wrong,
56 |     # because the sandbox creation should not modify the state outside of the
57 |     # sandbox... but this behavior is less annoying.
58 |     mkdir -p "${DISTDIR}"
59 |     mkdir -p "${PACKAGES}"
60 |     mkdir -p "${PBULK_LOG}"
61 |     mkdir -p "${PBULK_PACKAGES}"
62 |     mkdir -p "${PKGSRCDIR}"
63 | 
64 |     mkdir "${SANDBOX_ROOT}/pkg_comp"
65 |     mkdir "${SANDBOX_ROOT}/pkg_comp/bulklog"
66 |     mkdir "${SANDBOX_ROOT}/pkg_comp/distfiles"
67 |     mkdir "${SANDBOX_ROOT}/pkg_comp/packages"
68 |     mkdir "${SANDBOX_ROOT}/pkg_comp/packages/pbulk"
69 |     mkdir "${SANDBOX_ROOT}/pkg_comp/packages/pkg"
70 |     mkdir "${SANDBOX_ROOT}/pkg_comp/pkgsrc"
71 | 
72 |     # Invoke the user-provided hook, which is guaranteed to exist by the
73 |     # contents of sandbox.conf.pre.
74 |     pkg_comp_post_create_hook
75 | }
76 | 
77 | 
78 | # Mounts the pkg_comp-specific file systems within the sandbox.
79 | post_mount_hook() {
80 |     _pkg_comp_validate_variables
81 | 
82 |     sandbox_bindfs -o rw "${PBULK_LOG}" "${SANDBOX_ROOT}/pkg_comp/bulklog"
83 |     sandbox_bindfs -o rw "${DISTDIR}" "${SANDBOX_ROOT}/pkg_comp/distfiles"
84 |     sandbox_bindfs -o rw "${PACKAGES}" "${SANDBOX_ROOT}/pkg_comp/packages/pkg"
85 |     sandbox_bindfs -o rw "${PBULK_PACKAGES}" \
86 |         "${SANDBOX_ROOT}/pkg_comp/packages/pbulk"
87 | 
88 |     local mode=ro
89 |     if shtk_bool_check "${PKG_DEVELOPER}"; then
90 |         mode=rw
91 |     fi
92 |     sandbox_bindfs -o "${mode}" "${PKGSRCDIR}" "${SANDBOX_ROOT}/pkg_comp/pkgsrc"
93 | 
94 |     # Invoke the user-provided hook, which is guaranteed to exist by the
95 |     # contents of sandbox.conf.pre.
96 |     pkg_comp_post_mount_hook
97 | }
98 | 


--------------------------------------------------------------------------------
/pkg_comp4cron.sh:
--------------------------------------------------------------------------------
  1 | # Copyright 2012 Google Inc.
  2 | # All rights reserved.
  3 | #
  4 | # Redistribution and use in source and binary forms, with or without
  5 | # modification, are permitted provided that the following conditions are
  6 | # met:
  7 | #
  8 | # * Redistributions of source code must retain the above copyright
  9 | #   notice, this list of conditions and the following disclaimer.
 10 | # * Redistributions in binary form must reproduce the above copyright
 11 | #   notice, this list of conditions and the following disclaimer in the
 12 | #   documentation and/or other materials provided with the distribution.
 13 | # * Neither the name of Google Inc. nor the names of its contributors
 14 | #   may be used to endorse or promote products derived from this software
 15 | #   without specific prior written permission.
 16 | #
 17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 18 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 19 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 20 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 21 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 22 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 23 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 24 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 25 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 26 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 28 | 
 29 | # \file pkg_comp4cron.sh
 30 | # Entry point and main program logic.
 31 | #
 32 | # TODO(jmmv): This is a vile copy from the sysbuild4cron script provided
 33 | # by sysbuild.  We should remove such divergence in some way between the
 34 | # two packages.
 35 | 
 36 | shtk_import cli
 37 | 
 38 | 
 39 | # Paths to installed files.
 40 | #
 41 | # Can be overriden for test purposes only.
 42 | : ${PKG_COMP_BINDIR="__PKG_COMP_BINDIR__"}
 43 | 
 44 | 
 45 | # Sends an email notification upon a failure.
 46 | #
 47 | # \param logfile File containing the output of the failed command.
 48 | # \param recipient Who to send the email to.
 49 | # \param ... Command that failed.
 50 | pkg_comp4cron_email() {
 51 |     local logfile="${1}"; shift
 52 |     local recipient="${1}"; shift
 53 | 
 54 |     local maxlines=100
 55 |     {
 56 |         echo "The following command has failed:"
 57 |         echo
 58 |         echo "    $*"
 59 |         echo
 60 |         echo "The output of the failed command has been left in:"
 61 |         echo
 62 |         echo "    ${logfile}"
 63 |         echo
 64 |         echo "The last ${maxlines} of the log follow:"
 65 |         echo
 66 |         tail -n${maxlines} "${logfile}"
 67 |     } | mail -s "pkg_comp failure report" "${recipient}"
 68 | }
 69 | 
 70 | 
 71 | # Entry point to the program.
 72 | #
 73 | # \param ... Command-line arguments to be processed.
 74 | #
 75 | # \return An exit code to be returned to the user.
 76 | main() {
 77 |     local logdir="${HOME}/pkg_comp/log"
 78 |     local recipient="$(id -u -n)"
 79 | 
 80 |     while getopts ':l:r:' arg "${@}"; do
 81 |         case "${arg}" in
 82 |             l)  # Location of logs.
 83 |                 case "${OPTARG}" in
 84 |                     /*) logdir="${OPTARG}" ;;
 85 |                     *) logdir="$(pwd)/${OPTARG}" ;;
 86 |                 esac
 87 |                 ;;
 88 | 
 89 |             r)  # Email recipient.
 90 |                 recipient="${OPTARG}"
 91 |                 ;;
 92 | 
 93 |             :)
 94 |                 shtk_cli_usage_error "Missing argument to option -${OPTARG}"
 95 |                 ;;
 96 | 
 97 |             \?)
 98 |                 shtk_cli_usage_error "Unknown option -${OPTARG}"
 99 |                 ;;
100 |         esac
101 |     done
102 |     shift $((${OPTIND} - 1))
103 | 
104 |     set -- "${PKG_COMP_BINDIR}/pkg_comp" "${@}"
105 | 
106 |     if [ ! -d "${logdir}" ]; then
107 |         mkdir -p "$(dirname "${logdir}")" >/dev/null 2>/dev/null || true
108 |         mkdir "${logdir}" \
109 |             || shtk_cli_error "Failed to create directory '${logdir}'"
110 |     fi
111 | 
112 |     local exit_code=0
113 |     local logfile="${logdir}/$(shtk_cli_progname).$(date +%Y%m%d%H%M%S).log"
114 |     "${@}" >"${logfile}" 2>&1 \
115 |         || pkg_comp4cron_email "${logfile}" "${recipient}" "${@}"
116 | }
117 | 


--------------------------------------------------------------------------------
/git.subr:
--------------------------------------------------------------------------------
  1 | # Copyright 2017 Google Inc.
  2 | # All rights reserved.
  3 | #
  4 | # Redistribution and use in source and binary forms, with or without
  5 | # modification, are permitted provided that the following conditions are
  6 | # met:
  7 | #
  8 | # * Redistributions of source code must retain the above copyright
  9 | #   notice, this list of conditions and the following disclaimer.
 10 | # * Redistributions in binary form must reproduce the above copyright
 11 | #   notice, this list of conditions and the following disclaimer in the
 12 | #   documentation and/or other materials provided with the distribution.
 13 | # * Neither the name of Google Inc. nor the names of its contributors
 14 | #   may be used to endorse or promote products derived from this software
 15 | #   without specific prior written permission.
 16 | #
 17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 18 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 19 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 20 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 21 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 22 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 23 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 24 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 25 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 26 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 28 | 
 29 | # \file git.subr
 30 | # Utilities to invoke git, similar to shtk_cvs(3).
 31 | #
 32 | # This is an shtk module and, given its generic functionality and its
 33 | # similiarity to shtk_cvs(3), it should live in the shtk distribution.
 34 | # TODO(jmmv): Move this module to shtk once stabilized.
 35 | 
 36 | shtk_import cli
 37 | shtk_import process
 38 | 
 39 | 
 40 | git_fetch() {
 41 |     local url="${1}"; shift
 42 |     local branch="${1}"; shift
 43 |     local directory="${1}"; shift
 44 | 
 45 |     if [ -d "${directory}" ]; then
 46 |         git_update "${url}" "${branch}" "${directory}"
 47 |     else
 48 |         git_clone "${url}" "${branch}" "${directory}"
 49 |     fi
 50 | }
 51 | 
 52 | 
 53 | git_clone() {
 54 |     local url="${1}"; shift
 55 |     local branch="${1}"; shift
 56 |     local directory="${1}"; shift
 57 | 
 58 |     local rflag=
 59 |     [ -z "${tag}" ] || rflag="-r${tag}"
 60 | 
 61 |     [ ! -d "${directory}" ] || shtk_cli_error "Cannot clone into" \
 62 |         "${directory}; directory already exists"
 63 |     if ! shtk_process_run git clone -b "${branch}" "${url}" "${directory}"
 64 |     then
 65 |         rm -rf "${directory}"
 66 |         shtk_cli_error "Git clone failed"
 67 |     fi
 68 | }
 69 | 
 70 | 
 71 | git_update() {
 72 |     local url="${1}"; shift
 73 |     local branch="${1}"; shift
 74 |     local directory="${1}"; shift
 75 | 
 76 |     [ -d "${directory}" ] || shtk_cli_error "Cannot update ${directory};" \
 77 |         "directory does not exist"
 78 | 
 79 |     (
 80 |         cd "${directory}"
 81 | 
 82 |         shtk_process_run git fetch "${url}" "${branch}" \
 83 |             || shtk_cli_error "Git fetch failed"
 84 | 
 85 |         local modified=no
 86 |         [ -z "$(git status --porcelain 2>&1 | grep -vE '^(\?\?|!!)')" ] \
 87 |             || modified=yes
 88 | 
 89 |         # It is tempting to use "git stash save --all"... but depending on how
 90 |         # pkg_comp has been configured, the distfiles and the packages may be
 91 |         # located within the pkgsrc tree and these are not recorded in
 92 |         # .gitignore (because git is not authoritative).  Stashing these large
 93 |         # binary files would cause a lot of churn for little benefit.
 94 |         [ "${modified}" = no ] || \
 95 |             shtk_process_run git stash save \
 96 |                 || shtk_cli_error "Local changes detected but could not save" \
 97 |                 "them"
 98 | 
 99 |         # The pkgsrc repository's history often gets rewritten because the
100 |         # commit messages are amended in the master CVS repository.  This causes
101 |         # a simple pull to fail, so be more aggressive.
102 |         shtk_process_run git checkout "${branch}" \
103 |             || shtk_cli_error "Failed to switch to desired branch"
104 |         shtk_process_run git reset --hard FETCH_HEAD \
105 |             || shtk_cli_error "Failed to update branch to fetched head"
106 | 
107 |         [ "${modified}" = no ] || \
108 |             shtk_process_run git stash apply \
109 |                 || shtk_cli_error "Failed to restore local changes after" \
110 |                                   "branch update"
111 |     ) || exit  # Error messages reported by subshell so just propagate code.
112 | }
113 | 


--------------------------------------------------------------------------------
/pkg_comp4cron_test.sh:
--------------------------------------------------------------------------------
  1 | # Copyright 2012 Google Inc.
  2 | # All rights reserved.
  3 | #
  4 | # Redistribution and use in source and binary forms, with or without
  5 | # modification, are permitted provided that the following conditions are
  6 | # met:
  7 | #
  8 | # * Redistributions of source code must retain the above copyright
  9 | #   notice, this list of conditions and the following disclaimer.
 10 | # * Redistributions in binary form must reproduce the above copyright
 11 | #   notice, this list of conditions and the following disclaimer in the
 12 | #   documentation and/or other materials provided with the distribution.
 13 | # * Neither the name of Google Inc. nor the names of its contributors
 14 | #   may be used to endorse or promote products derived from this software
 15 | #   without specific prior written permission.
 16 | #
 17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 18 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 19 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 20 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 21 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 22 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 23 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 24 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 25 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 26 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 28 | 
 29 | # \file pkg_comp4cron_test.sh
 30 | # Integration tests for the pkg_comp4cron.sh script.
 31 | 
 32 | shtk_import unittest
 33 | 
 34 | 
 35 | # Creates a fake program that records its invocations for later processing.
 36 | #
 37 | # The fake program, when invoked, will append its arguments to a commands.log
 38 | # file in the test case's work directory.
 39 | #
 40 | # \param binary The path to the program to create.
 41 | # \param get_stdin Whether to capture stdin or not.
 42 | create_mock_binary() {
 43 |     local binary="${1}"; shift
 44 |     local get_stdin="${1}"; shift
 45 | 
 46 |     cat >"${binary}" <<EOF
 47 | #! /bin/sh
 48 | 
 49 | logfile="${HOME}/commands.log"
 50 | echo "Command: \${0##*/}" >>"\${logfile}"
 51 | for arg in "\${@}"; do
 52 |     echo "Arg: \${arg}" >>"\${logfile}"
 53 | done
 54 |     [ "${get_stdin}" = no ] || sed -e 's,^,stdin: ,' >>"\${logfile}"
 55 |     echo >>"\${logfile}"
 56 | EOF
 57 |     chmod +x "${binary}"
 58 | }
 59 | 
 60 | 
 61 | setup_mocks() {
 62 |     mkdir bin
 63 |     create_mock_binary bin/mail yes
 64 |     create_mock_binary bin/pkg_comp no
 65 |     PATH="$(pwd)/bin:${PATH}"
 66 |     PKG_COMP_BINDIR="$(pwd)/bin"; export PKG_COMP_BINDIR
 67 | }
 68 | 
 69 | 
 70 | shtk_unittest_add_test no_args
 71 | no_args_test() {
 72 |     setup_mocks
 73 |     assert_command pkg_comp4cron
 74 | 
 75 |     assert_file stdin commands.log <<EOF
 76 | Command: pkg_comp
 77 | 
 78 | EOF
 79 | }
 80 | 
 81 | 
 82 | shtk_unittest_add_test some_args
 83 | some_args_test() {
 84 |     setup_mocks
 85 |     assert_command pkg_comp4cron -- -k -Z foo bar
 86 | 
 87 |     assert_file stdin commands.log <<EOF
 88 | Command: pkg_comp
 89 | Arg: -k
 90 | Arg: -Z
 91 | Arg: foo
 92 | Arg: bar
 93 | 
 94 | EOF
 95 | }
 96 | 
 97 | 
 98 | shtk_unittest_add_test pkg_comp_fails
 99 | pkg_comp_fails_test() {
100 |     setup_mocks
101 |     for number in $(seq 150); do
102 |         echo "echo line ${number}" >>bin/pkg_comp
103 |     done
104 |     echo "exit 1" >>bin/pkg_comp
105 | 
106 |     assert_command pkg_comp4cron a
107 | 
108 |     name="$(cd pkg_comp/log && echo pkg_comp4cron.*.log)"
109 |     cat >expout <<EOF
110 | Command: pkg_comp
111 | Arg: a
112 | 
113 | Command: mail
114 | Arg: -s
115 | Arg: pkg_comp failure report
116 | Arg: ${USER}
117 | stdin: The following command has failed:
118 | stdin: 
119 | stdin:     $(pwd)/bin/pkg_comp a
120 | stdin: 
121 | stdin: The output of the failed command has been left in:
122 | stdin: 
123 | stdin:     $(pwd)/pkg_comp/log/${name}
124 | stdin: 
125 | stdin: The last 100 of the log follow:
126 | stdin: 
127 | EOF
128 |     for number in $(seq 51 150); do
129 |         echo "stdin: line ${number}" >>expout
130 |     done
131 |     echo >>expout
132 |     assert_file file:expout commands.log
133 | }
134 | 
135 | 
136 | shtk_unittest_add_test custom_flags
137 | custom_flags_test() {
138 |     setup_mocks
139 |     echo "exit 1" >>bin/pkg_comp
140 | 
141 |     assert_command pkg_comp4cron -l path/to/logs -r somebody@example.net
142 | 
143 |     name="$(cd path/to/logs && echo pkg_comp4cron.*.log)"
144 |     assert_file stdin commands.log <<EOF
145 | Command: pkg_comp
146 | 
147 | Command: mail
148 | Arg: -s
149 | Arg: pkg_comp failure report
150 | Arg: somebody@example.net
151 | stdin: The following command has failed:
152 | stdin: 
153 | stdin:     $(pwd)/bin/pkg_comp
154 | stdin: 
155 | stdin: The output of the failed command has been left in:
156 | stdin: 
157 | stdin:     $(pwd)/path/to/logs/${name}
158 | stdin: 
159 | stdin: The last 100 of the log follow:
160 | stdin: 
161 | 
162 | EOF
163 | }
164 | 
165 | 
166 | shtk_unittest_add_test capture_out_and_err
167 | capture_out_and_err_test() {
168 |     setup_mocks
169 |     echo "echo foo" >>bin/pkg_comp
170 |     echo "echo bar 1>&2" >>bin/pkg_comp
171 |     echo "exit 1" >>bin/pkg_comp
172 | 
173 |     assert_command pkg_comp4cron
174 | 
175 |     expect_file match:"stdin: foo" commands.log
176 |     expect_file match:"stdin: bar" commands.log
177 | }
178 | 
179 | 
180 | shtk_unittest_add_test unknown_flag
181 | unknown_flag_test() {
182 |     cat >experr <<EOF
183 | pkg_comp4cron: E: Unknown option -Z
184 | Type 'man pkg_comp4cron' for help
185 | EOF
186 |     assert_command -s exit:1 -e file:experr pkg_comp4cron -Z
187 | }
188 | 
189 | 
190 | shtk_unittest_add_test missing_argument
191 | missing_argument_test() {
192 |     cat >experr <<EOF
193 | pkg_comp4cron: E: Missing argument to option -l
194 | Type 'man pkg_comp4cron' for help
195 | EOF
196 |     assert_command -s exit:1 -e file:experr pkg_comp4cron -l
197 | }
198 | 


--------------------------------------------------------------------------------
/Makefile.am:
--------------------------------------------------------------------------------
  1 | # Copyright 2013 Google Inc.
  2 | # All rights reserved.
  3 | #
  4 | # Redistribution and use in source and binary forms, with or without
  5 | # modification, are permitted provided that the following conditions are
  6 | # met:
  7 | #
  8 | # * Redistributions of source code must retain the above copyright
  9 | #   notice, this list of conditions and the following disclaimer.
 10 | # * Redistributions in binary form must reproduce the above copyright
 11 | #   notice, this list of conditions and the following disclaimer in the
 12 | #   documentation and/or other materials provided with the distribution.
 13 | # * Neither the name of Google Inc. nor the names of its contributors
 14 | #   may be used to endorse or promote products derived from this software
 15 | #   without specific prior written permission.
 16 | #
 17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 18 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 19 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 20 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 21 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 22 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 23 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 24 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 25 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 26 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 28 | 
 29 | PHONY_TARGETS =
 30 | 
 31 | doc_DATA = AUTHORS CONTRIBUTING.md CONTRIBUTORS COPYING NEWS
 32 | noinst_DATA = INSTALL.md README.md
 33 | EXTRA_DIST = $(doc_DATA) INSTALL.md README.md
 34 | 
 35 | BUILD_SED = \
 36 | 	sed -e 's,__ATF_SH__,$(ATF_SH),g' \
 37 | 	    -e 's,__PKG_COMP_BASESYSCONFDIR__,$(sysconfdir),g' \
 38 | 	    -e 's,__PKG_COMP_BINDIR__,$(exec_prefix)/sbin,g' \
 39 | 	    -e 's,__PKG_COMP_DOCDIR__,$(docdir),g' \
 40 | 	    -e 's,__PKG_COMP_ETCDIR__,$(pkg_comp_confdir),g' \
 41 | 	    -e 's,__PKG_COMP_SHAREDIR__,$(pkgdatadir),g' \
 42 | 	    -e 's,__PKG_COMP_SHTK_MODULESDIR__,$(pkgdatadir)/shtk,g' \
 43 | 	    -e 's,__SANDBOXCTL__,$(SANDBOXCTL),g'
 44 | BUILD_SCRIPT = \
 45 | 	$(BUILD_SED) <"$(srcdir)/$${source}" | \
 46 | 	    $(SHTK_TOOL) build -o "$${target}" -
 47 | BUILD_FILE = \
 48 | 	$(BUILD_SED) <"$(srcdir)/$${source}" >"$${target}.tmp"; \
 49 | 	mv "$${target}.tmp" "$${target}"
 50 | BUILD_OLD_TEST = \
 51 | 	$(BUILD_FILE); \
 52 | 	chmod +x "$${target}"
 53 | BUILD_TEST = \
 54 | 	$(BUILD_SED) <"$(srcdir)/$${source}" | \
 55 | 	    $(SHTK_TOOL) build -m shtk_unittest_main -o "$${target}" -
 56 | 
 57 | sbin_SCRIPTS = pkg_comp
 58 | CLEANFILES = pkg_comp
 59 | EXTRA_DIST += pkg_comp.sh
 60 | pkg_comp: $(srcdir)/pkg_comp.sh
 61 | 	$(AM_V_GEN)source=pkg_comp.sh target=pkg_comp; $(BUILD_SCRIPT)
 62 | 
 63 | sbin_SCRIPTS += pkg_comp4cron
 64 | CLEANFILES += pkg_comp4cron
 65 | EXTRA_DIST += pkg_comp4cron.sh
 66 | pkg_comp4cron: $(srcdir)/pkg_comp4cron.sh
 67 | 	$(AM_V_GEN)source=pkg_comp4cron.sh target=pkg_comp4cron; $(BUILD_SCRIPT)
 68 | 
 69 | man_MANS = pkg_comp.8
 70 | CLEANFILES += pkg_comp.8
 71 | EXTRA_DIST += pkg_comp.8.in
 72 | pkg_comp.8: $(srcdir)/pkg_comp.8.in
 73 | 	$(AM_V_GEN)source=pkg_comp.8.in target=pkg_comp.8; $(BUILD_FILE)
 74 | 
 75 | dist_man_MANS  = pkg_comp.conf.5
 76 | dist_man_MANS += pkg_comp4cron.8
 77 | 
 78 | dist_pkg_comp_conf_DATA = extra.mk.conf
 79 | 
 80 | nodist_pkg_comp_conf_DATA = default.conf
 81 | CLEANFILES += default.conf
 82 | EXTRA_DIST += default.conf.in
 83 | default.conf: $(srcdir)/default.conf.in
 84 | 	$(AM_V_GEN)source=default.conf.in target=default.conf; $(BUILD_FILE)
 85 | 
 86 | nodist_pkg_comp_conf_DATA += sandbox.conf
 87 | CLEANFILES += sandbox.conf
 88 | EXTRA_DIST += sandbox.conf.in
 89 | sandbox.conf: $(srcdir)/sandbox.conf.in
 90 | 	$(AM_V_GEN)source=sandbox.conf.in target=sandbox.conf; $(BUILD_FILE)
 91 | 
 92 | dist_pkgdata_DATA = sandbox.conf.pre sandbox.conf.post
 93 | 
 94 | shtk_modulesdir = $(pkgdatadir)/shtk
 95 | dist_shtk_modules_DATA  = git.subr
 96 | dist_shtk_modules_DATA += pkgsrc.subr
 97 | 
 98 | if WITH_ATF
 99 | dist_pkgtests_DATA = Kyuafile
100 | 
101 | pkgtests_SCRIPTS = git_test
102 | CLEANFILES += git_test
103 | EXTRA_DIST += git_test.sh
104 | git_test: $(srcdir)/git_test.sh
105 | 	$(AM_V_GEN)source="git_test.sh" target=git_test; \
106 | 	$(BUILD_TEST)
107 | 
108 | pkgtests_SCRIPTS += pkgsrc_test
109 | CLEANFILES += pkgsrc_test
110 | EXTRA_DIST += pkgsrc_test.sh
111 | pkgsrc_test: $(srcdir)/pkgsrc_test.sh
112 | 	$(AM_V_GEN)source="pkgsrc_test.sh" target=pkgsrc_test; \
113 | 	$(BUILD_TEST)
114 | 
115 | pkgtests_SCRIPTS += pkg_comp_test
116 | CLEANFILES += pkg_comp_test
117 | EXTRA_DIST += pkg_comp_test.sh
118 | pkg_comp_test: $(srcdir)/pkg_comp_test.sh
119 | 	$(AM_V_GEN)source=pkg_comp_test.sh target=pkg_comp_test; \
120 | 	$(BUILD_OLD_TEST)
121 | 
122 | pkgtests_SCRIPTS += pkg_comp_inttest
123 | CLEANFILES += pkg_comp_inttest
124 | EXTRA_DIST += pkg_comp_inttest.sh
125 | pkg_comp_inttest: $(srcdir)/pkg_comp_inttest.sh
126 | 	$(AM_V_GEN)source=pkg_comp_inttest.sh target=pkg_comp_inttest; \
127 | 	$(BUILD_OLD_TEST)
128 | 
129 | pkgtests_SCRIPTS += pkg_comp4cron_test
130 | CLEANFILES += pkg_comp4cron_test
131 | EXTRA_DIST += pkg_comp4cron_test.sh
132 | pkg_comp4cron_test: $(srcdir)/pkg_comp4cron_test.sh
133 | 	$(AM_V_GEN)source=pkg_comp4cron_test.sh target=pkg_comp4cron_test; \
134 | 	$(BUILD_TEST)
135 | 
136 | if HAVE_KYUA
137 | CHECK_ENVIRONMENT  = PATH=$(abs_top_builddir):$${PATH}
138 | CHECK_ENVIRONMENT += PKG_COMP_ETCDIR=$(abs_top_srcdir)
139 | CHECK_ENVIRONMENT += PKG_COMP_SHAREDIR=$(abs_top_srcdir)
140 | CHECK_ENVIRONMENT += PKG_COMP_SHTK_MODULESDIR=$(abs_top_srcdir)
141 | INSTALLCHECK_ENVIRONMENT = PATH=$(prefix)/sbin:$${PATH}
142 | 
143 | # Allow the caller to override the configuration file passed to our test runs
144 | # below (or any other argument for that matter).
145 | KYUA_FLAGS ?=
146 | 
147 | check-local: check-kyua
148 | PHONY_TARGETS += check-kyua
149 | check-kyua:
150 | 	$(CHECK_ENVIRONMENT) $(TESTS_ENVIRONMENT) \
151 | 	    $(KYUA) $(KYUA_FLAGS) test \
152 | 	    --kyuafile="$$(cd '$(top_srcdir)' && echo "$$(pwd -P)/Kyuafile")" \
153 | 	    --build-root='$(top_builddir)' \
154 | 	    || { cd '$(top_srcdir)'; $(KYUA) report --verbose; false; }
155 | 
156 | installcheck-local: installcheck-kyua
157 | PHONY_TARGETS += installcheck-kyua
158 | installcheck-kyua:
159 | 	cd $(pkgtestsdir) && $(INSTALLCHECK_ENVIRONMENT) $(TESTS_ENVIRONMENT) \
160 | 	    $(KYUA) $(KYUA_FLAGS) test
161 | else  # HAVE_KYUA
162 | PHONY_TARGETS += missing-kyua
163 | missing-kyua:
164 | 	@echo "WARNING: kyua not found; no tests run"
165 | 
166 | check-local: missing-kyua
167 | installcheck-local: missing-kyua
168 | endif  # HAVE_KYUA
169 | else  # WITH_ATF
170 | dist-hook: forbid-dist
171 | PHONY_TARGETS += forbid-dist
172 | forbid-dist:
173 | 	@echo "Sorry; cannot make dist without atf."
174 | 	@false
175 | endif  # WITH_ATF
176 | 
177 | PHONY_TARGETS += clean-all
178 | clean-all:
179 | 	GIT="$(GIT)" $(SH) $(srcdir)/admin/clean-all.sh
180 | 
181 | .PHONY: $(PHONY_TARGETS)
182 | 


--------------------------------------------------------------------------------
/admin/bootstrap.sh:
--------------------------------------------------------------------------------
  1 | # Copyright 2017 Google Inc.
  2 | # All rights reserved.
  3 | #
  4 | # Redistribution and use in source and binary forms, with or without
  5 | # modification, are permitted provided that the following conditions are
  6 | # met:
  7 | #
  8 | # * Redistributions of source code must retain the above copyright
  9 | #   notice, this list of conditions and the following disclaimer.
 10 | # * Redistributions in binary form must reproduce the above copyright
 11 | #   notice, this list of conditions and the following disclaimer in the
 12 | #   documentation and/or other materials provided with the distribution.
 13 | # * Neither the name of Google Inc. nor the names of its contributors
 14 | #   may be used to endorse or promote products derived from this software
 15 | #   without specific prior written permission.
 16 | #
 17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 18 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 19 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 20 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 21 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 22 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 23 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 24 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 25 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 26 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 28 | 
 29 | # \file admin/bootstrap.sh
 30 | # Fetches all pkg_comp components and installs them.
 31 | #
 32 | # This file must be self-contained so that it can be downloaded from the
 33 | # repository and used to install pkg_comp and all necessary components on the
 34 | # supported platforms.
 35 | #
 36 | # The -d flag can be used to customize the location and version of some
 37 | # dependencies.  This flag exists purely to support testing this script
 38 | # against unreleased versions of those packages and should not be used in
 39 | # the general case.
 40 | 
 41 | 
 42 | # Base name of the running script.
 43 | ProgName="${0##*/}"
 44 | 
 45 | 
 46 | # Prints the given error message to stderr and exits.
 47 | #
 48 | # \param ... The message to print.
 49 | err() {
 50 |     echo "${ProgName}: E: $*" 1>&2
 51 |     exit 1
 52 | }
 53 | 
 54 | 
 55 | # Prints the given informational message to stderr.
 56 | #
 57 | # \param ... The message to print.
 58 | info() {
 59 |     echo "${ProgName}: I: $*" 1>&2
 60 | }
 61 | 
 62 | 
 63 | # Downloads a file specified by an URL.
 64 | #
 65 | # \param url The file to download.
 66 | # \param file Path to where to save the downloaded file.
 67 | download() {
 68 |     local url="${1}"; shift
 69 |     local file="${1}"; shift
 70 | 
 71 |     info "Downloading ${url}"
 72 |     local done=no
 73 | 
 74 |     case "${url}" in
 75 |         /*)
 76 |             cp "${url}" "${file}" || err "Failed to copy ${url}"
 77 |             done=yes
 78 |             ;;
 79 |     esac
 80 | 
 81 |     if [ "${done}" = no ]; then
 82 |         case "$(uname -s)" in
 83 |             FreeBSD)
 84 |                 fetch -o "${file}" "${url}" || err "Failed to download ${url}"
 85 |                 done=yes
 86 |                 ;;
 87 | 
 88 |             NetBSD)
 89 |                 ftp -o "${file}" "${url}" || err "Failed to download ${url}"
 90 |                 done=yes
 91 |                 ;;
 92 |         esac
 93 |     fi
 94 | 
 95 |     if [ "${done}" = no ]; then
 96 |         if which curl >/dev/null; then
 97 |             curl -L "${url}" >"${file}.tmp" || err "Failed to download ${url}"
 98 |             mv "${file}.tmp" "${file}"
 99 |             done=yes
100 |         elif which wget >/dev/null; then
101 |             wget -O "${file}" "${file}" || err "Failed to download ${url}"
102 |             done=yes
103 |         fi
104 |     fi
105 | 
106 |     [ "${done}" = yes ] \
107 |         || err "Sorry; don't know how to fetch files on your system"
108 | }
109 | 
110 | 
111 | # Builds and installs a package.
112 | #
113 | # \param workdir Temporary directory in which to find distfiles, extract them,
114 | #     and run the builds.
115 | # \param distname Basename of the distfile.  The distfile must exist within the
116 | #     given work directory and end in .tar.gz.
117 | # \param ... Extra arguments to the configure script.
118 | build() {
119 |     local workdir="${1}"; shift
120 |     local distname="${1}"; shift
121 | 
122 |     info "Extracting ${distname}"
123 |     tar -xz -C "${workdir}" -f "${workdir}/${distname}.tar.gz" \
124 |         || err "Failed to extract ${distname}"
125 | 
126 |     info "Configuring and building ${distname}"
127 |     (
128 |         cd "${workdir}/${distname}" || exit
129 |         ./configure "${@}" || exit
130 |         make || exit
131 |         make install || exit
132 |     ) || err "Failed to install ${distname}; see output for details"
133 | }
134 | 
135 | 
136 | # Prints a usage error and a syntax summary.
137 | usage_error() {
138 |     echo "${ProgName}:" "${@}" 1>&2
139 |     echo "Usage: ${ProgName} [-d PKGNAME=URL.tar.gz] [prefix]" 1>&2
140 |     exit 1
141 | }
142 | 
143 | 
144 | # Program's entry point.
145 | main() {
146 |     local pkg_comp_url="https://github.com/jmmv/pkg_comp/releases/download/pkg_comp-2.1/pkg_comp-2.1.tar.gz"
147 |     local sandboxctl_url="https://github.com/jmmv/sandboxctl/releases/download/sandboxctl-1.1/sandboxctl-1.1.tar.gz"
148 |     local shtk_url="https://github.com/jmmv/shtk/releases/download/shtk-1.7/shtk-1.7.tar.gz"
149 | 
150 |     local OPTIND  # Cope with bash failing to reinitialize getopt.
151 |     while getopts ':d:' arg "${@}"; do
152 |         case "${arg}" in
153 |             d)  # Distfile override of the form PKGNAME=URL.tar.gz
154 |                 local pkgname="${OPTARG%%=*}"
155 |                 local url="${OPTARG#*=}"
156 | 
157 |                 case "${pkgname}" in
158 |                     pkg_comp) pkg_comp_url="${url}" ;;
159 |                     sandboxctl) sandboxctl_url="${url}" ;;
160 |                     shtk) shtk_url="${url}" ;;
161 |                     *) usage_error "Unknown pkgname ${pkgname} in -d flag" ;;
162 |                 esac
163 |                 ;;
164 |             \?)
165 |                 usage_error "Unknown option -${OPTARG} in build"
166 |                 ;;
167 |         esac
168 |     done
169 |     shift $((${OPTIND} - 1))
170 | 
171 |     local prefix
172 |     if [ ${#} -eq 0 ]; then
173 |         prefix=/usr/local
174 |     elif [ ${#} -eq 1 ]; then
175 |         prefix="${1}"
176 |     else
177 |         usage_error "Invalid number of arguments"
178 |     fi
179 | 
180 |     local pkg_comp_distname="${pkg_comp_url##*/}"
181 |     local sandboxctl_distname="${sandboxctl_url##*/}"
182 |     local shtk_distname="${shtk_url##*/}"
183 | 
184 |     info "Bootstrapping pkg_comp installation into ${prefix}"
185 | 
186 |     local tempdir
187 |     tempdir="$(mktemp -d "${TMPDIR:-/tmp}/${ProgName}.XXXXXX" 2>/dev/null)" \
188 |         || err "Failed to create temporary directory"
189 |         echo $tempdir
190 |     trap "rm -rf '${tempdir}'" EXIT
191 | 
192 |     PATH="${prefix}/bin:${prefix}/sbin:${PATH}"
193 |     export PKG_CONFIG_PATH="${prefix}/lib/pkgconfig"
194 | 
195 |     if ! which pkg-config >/dev/null >/dev/null || ! which pkgconf >/dev/null
196 |     then
197 |         download \
198 |             https://distfiles.dereferenced.org/pkgconf/pkgconf-1.3.0.tar.gz \
199 |             "${tempdir}/pkgconf-1.3.0.tar.gz"
200 |     fi
201 | 
202 |     case "$(uname -s)" in
203 |         Darwin)
204 |             download "http://bindfs.org/downloads/bindfs-1.13.6.tar.gz" \
205 |                 "${tempdir}/bindfs-1.13.6.tar.gz" || exit 1
206 |             PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:/usr/local/lib/pkgconfig"
207 |             ;;
208 |     esac
209 | 
210 |     download "${pkg_comp_url}" "${tempdir}/${pkg_comp_distname}"
211 |     download "${sandboxctl_url}" "${tempdir}/${sandboxctl_distname}"
212 |     download "${shtk_url}" "${tempdir}/${shtk_distname}"
213 | 
214 |     if [ -e "${tempdir}/pkgconf-1.3.0.tar.gz" ]; then
215 |         build "${tempdir}" pkgconf-1.3.0 --prefix="${tempdir}/tools" || exit 1
216 |         export PKG_CONFIG="${tempdir}/tools/bin/pkgconf"
217 |     fi
218 |     if [ -e "${tempdir}/bindfs-1.13.6.tar.gz" ]; then
219 |         build "${tempdir}" bindfs-1.13.6 --prefix="${prefix}" || exit 1
220 |     fi
221 |     build "${tempdir}" "${shtk_distname%.tar.gz}" \
222 |         --prefix="${prefix}" SHTK_SHELL=/bin/sh || exit 1
223 |     build "${tempdir}" "${sandboxctl_distname%.tar.gz}" \
224 |         --prefix="${prefix}" --with-atf=no BINDFS="${prefix}/bin/bindfs" \
225 |         || exit 1
226 |     build "${tempdir}" "${pkg_comp_distname%.tar.gz}" \
227 |         --prefix="${prefix}" --with-atf=no || exit 1
228 | 
229 |     info "Bootstrapping successful"
230 |     info "pkg_comp is available at: ${prefix}/sbin/pkg_comp"
231 | }
232 | 
233 | 
234 | main "${@}"
235 | 


--------------------------------------------------------------------------------
/INSTALL.md:
--------------------------------------------------------------------------------
  1 | # Installation instructions
  2 | 
  3 | Getting pkg_comp up and running is easy, and the process does not require a
  4 | pre-existing pkgsrc installation.  This is intentional given that pkg_comp
  5 | is intended to simplify interactions with pkgsrc, so cyclic dependencies would
  6 | be suboptimal.
  7 | 
  8 | You can choose to use an installer package, the installer script, source or
  9 | binary packages for your operating system, or to build from source.
 10 | 
 11 | ## Using the generic installer
 12 | 
 13 | **Run:**
 14 | 
 15 |     curl -L https://raw.githubusercontent.com/jmmv/pkg_comp/master/admin/bootstrap.sh | /bin/sh /dev/stdin
 16 | 
 17 | This will fetch and run a script that downloads pkg_comp and its dependencies as
 18 | source packages, builds all of them, and installs the results under the
 19 | `/usr/local/` prefix.  The resulting installation is barebones: i.e. the
 20 | packages are installed exactly as distributed by the upstream distribution
 21 | files.  No post-install configuration is executed, which means you will have to
 22 | set everything up by yourself; see `pkg_comp(8)` to get started.
 23 | 
 24 | You will need the `pkg-config` tool on your system for this to work, but
 25 | that should be the only necessary dependency.
 26 | 
 27 | ## Using the macOS installer
 28 | 
 29 | **Download and run the
 30 | [pkg_comp-2.1-20190114-macos.pkg](../../releases/download/pkg_comp-2.1/pkg_comp-2.1-20190114-macos.pkg)
 31 | installer.**  As a prerequisite on macOS, you will also have to **[download
 32 | and install OSXFUSE 3](https://osxfuse.github.io/).**
 33 | 
 34 | **Read
 35 | [Easy pkgsrc on macOS with pkg_comp 2.0](http://julio.meroh.net/2017/02/pkg_comp-2.0-tutorial-macos.html)
 36 | for a tutorial on this package.**
 37 | 
 38 | This is a highly-customized installation of pkg_comp intended to simplify the
 39 | use of pkgsrc on this platform.  Because of this, the installer is very
 40 | prescriptive about the configuration and the location of the installed files.
 41 | In particular, the package will:
 42 | 
 43 | *   Install pkg_comp as `/usr/local/sbin/pkg_comp`.
 44 | *   Place configuration files under `/usr/local/etc/pkg_comp/`.
 45 | *   Create `/var/pkg_comp/` to host the pkgsrc tree and the built packages.
 46 | *   Default the installation of the built packages to `/opt/pkg`.
 47 | *   Configure a daily cron job, as root, to build a fresh set of packages.
 48 | 
 49 | All you need to do to get started is modify
 50 | `/usr/local/etc/pkg_comp/list.txt` to indicate which packages you would
 51 | like built and they will eventually show up under
 52 | `/var/pkg_comp/packages/All/`.
 53 | 
 54 | You can use `/usr/local/libexec/pkg_comp/uninstall.sh` to undo the actions
 55 | performed by this package.  The contents of `/var/pkg_comp/` will be left
 56 | behind.  Feel free to destroy that directory if you do not need any of the
 57 | packages you previously built nor the pkgsrc tree.
 58 | 
 59 | ## Using operating-system packages
 60 | 
 61 | The following packages are known to exist:
 62 | 
 63 | *   **pkgsrc** (for NetBSD): `pkgtools/pkg_comp` and `pkgtools/pkg_comp-cron`.
 64 |     *   **Read
 65 |         [Keeping NetBSD up-to-date with pkg_comp 2.0](http://julio.meroh.net/2017/02/pkg_comp-2.0-tutorial-netbsd.html)**
 66 |         for details on how to use the `pkg_comp-cron` package.
 67 | 
 68 | ## Building from source
 69 | 
 70 | Download the
 71 | [pkg_comp-2.1.tar.gz](../../releases/download/pkg_comp-2.1/pkg_comp-2.1.tar.gz)
 72 | distribution file.
 73 | 
 74 | pkg_comp uses the GNU Automake and GNU Autoconf utilities as its build
 75 | system.  These are used only when building the package from the source
 76 | code tree.  If you want to install pkg_comp from a prebuilt package
 77 | provided by your operating system, you do not need to read this
 78 | document.
 79 | 
 80 | For the impatient:
 81 | 
 82 |     $ ./configure
 83 |     $ make
 84 |     $ make check
 85 |     Gain root privileges
 86 |     # make install
 87 |     Drop root privileges
 88 |     $ make installcheck
 89 | 
 90 | Or alternatively, install as a regular user into your home directory:
 91 | 
 92 |     $ ./configure --prefix ~/local
 93 |     $ make
 94 |     $ make check
 95 |     $ make install
 96 |     $ make installcheck
 97 | 
 98 | ### Dependencies
 99 | 
100 | To build and use pkg_comp successfully you need:
101 | 
102 | * shtk 1.7 or greater.
103 | * sandboxctl 1.0 or greater.
104 | * pkg-config.
105 | 
106 | Optionally, if you want to build and run the tests (recommended), you
107 | need:
108 | 
109 | * ATF 0.17 or greater.
110 | * Kyua 0.6 or greater.
111 | 
112 | If you are building pkg_comp from the code on the repository, you will
113 | also need the following tools:
114 | 
115 | * GNU Autoconf.
116 | * GNU Automake.
117 | 
118 | ### Regenerating the build system
119 | 
120 | This is not necessary if you are building from a formal release
121 | distribution file.
122 | 
123 | On the other hand, if you are building pkg_comp from code extracted
124 | from the repository, you must first regenerate the files used by the
125 | build system.  You will also need to do this if you modify configure.ac,
126 | Makefile.am or any of the other build system files.  To do this, simply
127 | run:
128 | 
129 |     $ autoreconf -i -s
130 | 
131 | If ATF and/or shtk are installed in a different prefix than Autoconf,
132 | you will also need to tell autoreconf where the ATF and shtk M4 macros
133 | are located.  Otherwise, the configure script will be incomplete and
134 | will show confusing syntax errors mentioning, for example, ATF_CHECK_SH.
135 | To fix this, you have to run autoreconf in the following manner,
136 | replacing '<atf-prefix>' and '<shtk-prefix>' with the appropriate path:
137 | 
138 |     $ autoreconf -i -s -I <atf-prefix>/share/aclocal \
139 |       -I <shtk-prefix>/share/aclocal
140 | 
141 | ### General build procedure
142 | 
143 | To build and install the source package, you must follow these steps:
144 | 
145 | 1. Configure the sources to adapt to your operating system.  This is
146 |    done using the 'configure' script located on the sources' top
147 |    directory, and it is usually invoked without arguments unless you
148 |    want to change the installation prefix.  More details on this
149 |    procedure are given on a later section.
150 | 
151 | 2. Build the sources to generate the binaries and scripts.  Simply run
152 |    'make' on the sources' top directory after configuring them.  No
153 |    problems should arise.
154 | 
155 | 3. Install the library by running 'make install'.  You may need to
156 |    become root to issue this step.
157 | 
158 | 4. Issue any manual installation steps that may be required.  These are
159 |    described later in their own section.
160 | 
161 | 5. Check that the installed library works by running 'make
162 |    installcheck'.  You do not need to be root to do this.
163 | 
164 | ### Configuration flags
165 | 
166 | The most common, standard flags given to 'configure' are:
167 | 
168 | * --prefix=directory
169 |   Possible values: Any path
170 |   Default: /usr/local
171 | 
172 |   Specifies where the library (scripts and all associated files) will
173 |   be installed.
174 | 
175 | * --sysconfdir=directory
176 |   Possible values: Any path
177 |   Default: /usr/local/etc
178 | 
179 |   Specifies where the installed programs will look for configuration
180 |   files.  '/pkg_comp' will be appended to the given path unless
181 |   PKG_COMP_CONFSUBDIR is redefined as explained later on.
182 | 
183 | * --help
184 |   Shows information about all available flags and exits immediately,
185 |   without running any configuration tasks.
186 | 
187 | The following environment variables are specific to pkg_comp's
188 | 'configure' script:
189 | 
190 | * PKG_COMP_CONFSUBDIR
191 |   Possible values: empty, a relative path.
192 |   Default: pkg_comp.
193 | 
194 |   Specifies the subdirectory of the configuration directory (given by
195 |   the --sysconfdir argument) under which pkg_comp will search for its
196 |   configuration files.
197 | 
198 | The following flags are specific to pkg_comp's 'configure' script:
199 | 
200 | * --with-atf
201 |   Possible values: yes, no, auto.
202 |   Default: auto.
203 | 
204 |   Enables usage of ATF to build (and later install) the tests.
205 | 
206 |   Setting this to 'yes' causes the configure script to look for ATF
207 |   unconditionally and abort if not found.  Setting this to 'auto' lets
208 |   configure perform the best decision based on availability of ATF.
209 |   Setting this to 'no' explicitly disables ATF usage.
210 | 
211 |   When support for tests is enabled, the build process will generate the
212 |   test programs and will later install them into the tests tree.
213 |   Running 'make check' or 'make installcheck' from within the source
214 |   directory will cause these tests to be run with Kyua (assuming it is
215 |   also installed).
216 | 
217 | ### Run the tests!
218 | 
219 | Lastly, after a successful installation (and assuming you built the
220 | sources with support for ATF), you should periodically run the tests
221 | from the final location to ensure things remain stable.  Do so as
222 | follows:
223 | 
224 |     $ kyua test -k /usr/local/tests/pkg_comp/Kyuafile
225 | 
226 | And if you see any tests fail, do not hesitate to report them in:
227 | 
228 |     https://github.com/jmmv/pkg_comp/issues/
229 | 
230 | Thank you!
231 | 


--------------------------------------------------------------------------------
/pkg_comp.conf.5:
--------------------------------------------------------------------------------
  1 | .\" Copyright 2013 Google Inc.
  2 | .\" All rights reserved.
  3 | .\"
  4 | .\" Redistribution and use in source and binary forms, with or without
  5 | .\" modification, are permitted provided that the following conditions are
  6 | .\" met:
  7 | .\"
  8 | .\" * Redistributions of source code must retain the above copyright
  9 | .\"   notice, this list of conditions and the following disclaimer.
 10 | .\" * Redistributions in binary form must reproduce the above copyright
 11 | .\"   notice, this list of conditions and the following disclaimer in the
 12 | .\"   documentation and/or other materials provided with the distribution.
 13 | .\" * Neither the name of Google Inc. nor the names of its contributors
 14 | .\"   may be used to endorse or promote products derived from this software
 15 | .\"   without specific prior written permission.
 16 | .\"
 17 | .\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 18 | .\" "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 19 | .\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 20 | .\" A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 21 | .\" OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 22 | .\" SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 23 | .\" LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 24 | .\" DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 25 | .\" THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 26 | .\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 27 | .\" OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 28 | .Dd January 3, 2019
 29 | .Dt PKG_COMP.CONF 5
 30 | .Os
 31 | .Sh NAME
 32 | .Nm pkg_comp.conf
 33 | .Nd configuration file format for the pkg_comp tool
 34 | .Sh DESCRIPTION
 35 | Configuration files for
 36 | .Xr pkg_comp 8
 37 | are plain shell scripts that define a set of recognized variables and can
 38 | optionally fill in a set of hooks provided as shell functions.
 39 | .Ss Configuration variables
 40 | As scripts, configuration files can perform any magic they desire to deduce
 41 | the value of configuration variables.
 42 | .Pp
 43 | The following variables configure general settings:
 44 | .Bl -tag -width SANDBOX_CONFFILE
 45 | .It Va AUTO_PACKAGES
 46 | Whitespace-separated list of package names that will be automatically built
 47 | by the
 48 | .Sq auto
 49 | command unless package names are provided on the command line.
 50 | This is akin to pbulk's
 51 | .Dq limited list
 52 | file contents, specifying the subset of packages that are interesting for
 53 | this automated build.
 54 | .Pp
 55 | The packages to build can be provided either as bare names, such as
 56 | .Sq tmux ,
 57 | or as category/package pairs, such as
 58 | .Sq misc/tmux .
 59 | .It Va SANDBOX_CONFFILE
 60 | Absolute path to a file in the host that contains the
 61 | .Xr sandboxctl 8
 62 | configuration to use for the build of packages.
 63 | .Pp
 64 | The contents of this file are slightly different than what
 65 | .Xr sandboxctl 8
 66 | accepts.
 67 | In particular, if you want to define
 68 | .Xr sandboxctl 8 Ns 's
 69 | own
 70 | .Fn post_create_hook
 71 | and
 72 | .Fn post_mount_hook
 73 | hooks, you must define
 74 | .Fn pkg_comp_post_create_hook
 75 | and
 76 | .Fn pkg_comp_post_mount_hook
 77 | instead.
 78 | Any changes to the former two will be discarded.
 79 | .Pp
 80 | Default: not defined.
 81 | .El
 82 | .Pp
 83 | The following variables configure how to fetch pkgsrc from a version control
 84 | system:
 85 | .Bl -tag -width SANDBOX_CONFFILE
 86 | .It Va CVS_ROOT
 87 | Location of the CVS root from which to check out or update the pkgsrc
 88 | module.
 89 | .Pp
 90 | This variable is only used if
 91 | .Va FETCH_VCS
 92 | is set to
 93 | .Sq cvs .
 94 | .Pp
 95 | Default:
 96 | .Sq :ext:anoncvs@anoncvs.NetBSD.org:/cvsroot .
 97 | .It Va CVS_TAG
 98 | CVS tag to use during checkouts or updates of the pkgsrc module.
 99 | .Pp
100 | This variable is only used if
101 | .Va FETCH_VCS
102 | is set to
103 | .Sq cvs .
104 | .Pp
105 | Default: not defined.
106 | .It Va FETCH_VCS
107 | Determines the version control system to use to fetch the pkgsrc tree.
108 | Can be one of
109 | .Sq cvs
110 | or
111 | .Sq git .
112 | The requested tool must be installed on the system.
113 | .Pp
114 | Default:
115 | .Sq cvs .
116 | .It Va GIT_BRANCH
117 | Git branch to use during clones or updates of the pkgsrc working copy.
118 | .Pp
119 | This variable is only used if
120 | .Va FETCH_VCS
121 | is set to
122 | .Sq git .
123 | .Pp
124 | Default:
125 | .Sq trunk .
126 | .It Va GIT_URL
127 | Git remote URL specification to use during clones or updates of the pkgsrc
128 | working copy.
129 | .Pp
130 | This variable is only used if
131 | .Va FETCH_VCS
132 | is set to
133 | .Sq git .
134 | .Pp
135 | Default:
136 | .Sq https://github.com/NetBSD/pkgsrc.git .
137 | .El
138 | .Pp
139 | The following variables configure the pkgsrc environment on the host (i.e
140 | .Em Ns . outside
141 | of the sandbox):
142 | .Bl -tag -width SANDBOX_CONFFILE
143 | .It Va DISTDIR
144 | Absolute path to the directory in the host that holds the distribution
145 | files for pkgsrc (i.e. all the source files downloaded during the build of
146 | packages).
147 | .Pp
148 | This directory is mounted read-write within the sandbox to allow reuse of
149 | the immutable distribution files across rebuilds of the sandbox.
150 | .Pp
151 | Default:
152 | .Pa /usr/pkgsrc/distfiles .
153 | .It Va PACKAGES
154 | Absolute path to the directory in the host that will hold the built binary
155 | packages and bootstrap kit for the target environment.
156 | .Pp
157 | Default:
158 | .Pa /usr/pkgsrc/packages .
159 | .It Va PBULK_LOG
160 | Absolute path to the directory in the host that will hold the logs of the
161 | pbulk build.
162 | The contents of this directory are wiped at the beginning of each
163 | build.
164 | .Pp
165 | Default:
166 | .Pa /usr/pkgsrc/packages/log .
167 | .It Va PBULK_PACKAGES
168 | Absolute path to the directory in the host that will hold the built binary
169 | packages and bootstrap kit for the pbulk installation in the sandbox.
170 | .Pp
171 | It does not make much sense to use these files outside of the sandbox because
172 | they are hardcoded to match the internal needs of
173 | .Xr pkg_comp 8
174 | (e.g. the paths used in these binary packages are not customizable).
175 | However,
176 | .Xr pkg_comp 8
177 | will reuse these files on subsequent rebuilds of the sandbox for speed, so you
178 | may want to keep them around.
179 | .Pp
180 | Default:
181 | .Pa /usr/pkgsrc/packages/pbulk .
182 | .It Va PKGSRCDIR
183 | Absolute path to the directory in the host that contains or will contain the
184 | pkgsrc tree.
185 | .Pp
186 | If you want
187 | .Xr pkg_comp 8
188 | to perform an update of this directory before every build, you will need
189 | write access to it.
190 | Otherwise, you can use a read-only directory.
191 | .Pp
192 | Default:
193 | .Pa /usr/pkgsrc .
194 | .It Va UPDATE_SOURCES
195 | Whether to perform an update of the source tree before every build or not.
196 | .Pp
197 | Default:
198 | .Sq true .
199 | .El
200 | .Pp
201 | The following variables configure the pkgsrc environment within the sandbox
202 | (i.e. how the packages will be built):
203 | .Bl -tag -width SANDBOX_CONFFILE
204 | .It Va EXTRA_MKCONF
205 | Absolute path to an optional file in the host that contains additional
206 | .Xr mk.conf 5
207 | contents to use for the build of packages within the sandbox.
208 | You can use this to configure settings like
209 | .Va ACCEPTABLE_LICENSES
210 | or
211 | .Va PKG_DEFAULT_OPTIONS .
212 | .Pp
213 | Default: not defined.
214 | .It Va LOCALBASE
215 | Absolute path to the directory where the packages will be installed.
216 | .Pp
217 | Default:
218 | .Pa /usr/pkg .
219 | .It Va NJOBS
220 | Number of
221 | .Xr make 1
222 | jobs to use during the build of packages.
223 | .Pp
224 | Default: number of CPUs in the machine.
225 | .It Va PKG_DBDIR
226 | Absolute path to the directory where
227 | .Xr pkg_add 8
228 | et. al. will maintain the database of installed directories.
229 | .Pp
230 | Default:
231 | .Pa /usr/pkg/libdata/pkgdb .
232 | .It Va PKG_DEVELOPER
233 | Enables development of pkgsrc packages within the sandbox via the
234 | .Sq sandbox-shell
235 | subcommand.
236 | This causes this same variable to be set in
237 | .Xr mk.conf 5
238 | and mounts pkgsrc in read/write mode.
239 | .Pp
240 | Default:
241 | .Pa no .
242 | .It Va SYSCONFDIR
243 | Absolute path to the directory in which to install managed configuration
244 | files.
245 | .Pp
246 | Default:
247 | .Pa /etc .
248 | .It Va VARBASE
249 | Absolute path to the directory that holds mutable data.
250 | .Pp
251 | Default:
252 | .Pa /var .
253 | .El
254 | .Ss Hooks
255 | Hooks are user-provided routines invoked at particular points during the
256 | build process.
257 | These are defined as shell functions and have access to the configuration
258 | variables in the file.
259 | Hooks must return a success error code and they cannot contain unprotected
260 | shell commands.
261 | In other words, they are run under a shell configured with
262 | .Sq set -e .
263 | .Pp
264 | The following hooks can be defined:
265 | .Bl -tag -width post_build_hookXX
266 | .It Fn post_fetch_hook
267 | Function executed right after all source trees have been updated by the
268 | .Sq fetch
269 | command.
270 | .El
271 | .Sh SEE ALSO
272 | .Xr sandbox.conf 5 ,
273 | .Xr pkg_comp 8
274 | 


--------------------------------------------------------------------------------
/admin/make-macos-pkg.sh:
--------------------------------------------------------------------------------
  1 | # Copyright 2017 Google Inc.
  2 | # All rights reserved.
  3 | #
  4 | # Redistribution and use in source and binary forms, with or without
  5 | # modification, are permitted provided that the following conditions are
  6 | # met:
  7 | #
  8 | # * Redistributions of source code must retain the above copyright
  9 | #   notice, this list of conditions and the following disclaimer.
 10 | # * Redistributions in binary form must reproduce the above copyright
 11 | #   notice, this list of conditions and the following disclaimer in the
 12 | #   documentation and/or other materials provided with the distribution.
 13 | # * Neither the name of Google Inc. nor the names of its contributors
 14 | #   may be used to endorse or promote products derived from this software
 15 | #   without specific prior written permission.
 16 | #
 17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 18 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 19 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 20 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 21 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 22 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 23 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 24 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 25 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 26 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 28 | 
 29 | # \file admin/make-macos-pkg.sh
 30 | # Builds a self-installer package for macOS.
 31 | #
 32 | # This script must be run with root privileges because we install the
 33 | # software under /usr/local.
 34 | #
 35 | # Any arguments passed to this script are delegated to bootstrap.sh, which
 36 | # makes it possible to generate packages for unreleased versions of pkg_comp
 37 | # and its dependencies.
 38 | #
 39 | # TODO(jmmv): A lot of the logic in this script should be moved into either
 40 | # separate files or into pkg_comp proper.  For example, the cron job
 41 | # manipulation code may belong into pkg_comp as user-facing commands, and
 42 | # the list.txt parsing should belong in the default configuration.
 43 | 
 44 | 
 45 | # Directory name of the running script.
 46 | DirName="$(dirname "${0}")"
 47 | 
 48 | 
 49 | # Base name of the running script.
 50 | ProgName="${0##*/}"
 51 | 
 52 | 
 53 | # Prints the given error message to stderr and exits.
 54 | #
 55 | # \param ... The message to print.
 56 | err() {
 57 |     echo "${ProgName}: E: $*" 1>&2
 58 |     exit 1
 59 | }
 60 | 
 61 | 
 62 | # Prints the given informational message to stderr.
 63 | #
 64 | # \param ... The message to print.
 65 | info() {
 66 |     echo "${ProgName}: I: $*" 1>&2
 67 | }
 68 | 
 69 | 
 70 | # Creates the package scripts.
 71 | #
 72 | # \param dir Directory in which to store the scripts.
 73 | write_scripts() {
 74 |     local dir="${1}"; shift
 75 | 
 76 |     mkdir -p "${dir}"
 77 |     cat >"${dir}/postinstall" <<EOF
 78 | #! /bin/sh
 79 | 
 80 | for pkg in sandboxctl pkg_comp; do
 81 |     for eg in /usr/local/share/examples/\${pkg}/*; do
 82 |         real="/usr/local/etc/\${pkg}/\${eg##*/}"
 83 |         if [ -e "\${real}" ]; then
 84 |             cmp -s "\${eg}" "\${real}" || cp "\${eg}" "\${real}.new"
 85 |         else
 86 |             cp "\${eg}" "\${real}"
 87 |         fi
 88 |     done
 89 | done
 90 | 
 91 | tempfile="\$(mktemp "\${TMPDIR:-/tmp}/pkg_comp.XXXXXX")"
 92 | if ! crontab -u root -l >>"\${tempfile}"; then
 93 |     cat >>"\${tempfile}" <<_EOF_
 94 | PATH=/usr/bin:/usr/sbin:/bin:/sbin
 95 | SHELL=/bin/sh
 96 | 
 97 | # Cheatsheet: minute hour day-of-month month day-of-week(0,7=Sun)
 98 | _EOF_
 99 | fi
100 | 
101 | if ! grep /usr/local/sbin/pkg_comp4cron "\${tempfile}" >/dev/null; then
102 |     echo "@daily /usr/local/sbin/pkg_comp4cron -l /var/pkg_comp/log" \
103 |         "-- -c /usr/local/etc/pkg_comp/default.conf auto" \
104 |         >>"\${tempfile}"
105 |     crontab -u root - <"\${tempfile}"
106 | fi
107 | rm -f "\${tempfile}"
108 | EOF
109 |     chmod +x "${dir}/postinstall"
110 | }
111 | 
112 | 
113 | # Sets a variable in a configuration file.
114 | #
115 | # \param file Path to the file to edit.
116 | # \param var Variable to set.
117 | # \param value Value to set the variable to.
118 | edit_config() {
119 |     local file="${1}"; shift
120 |     local var="${1}"; shift
121 |     local value="${1}"; shift
122 | 
123 |     sed -E "s,^#?${var}=.*$,${var}=\"${value}\",g" <"${file}" >"${file}.new"
124 |     mv "${file}.new" "${file}"
125 | }
126 | 
127 | 
128 | # Modifies the fresh pkg_comp installation for our packaging needs.
129 | #
130 | # \param root Path to the new file system root used to build the package.
131 | configure_root() {
132 |     local root="${1}"; shift
133 | 
134 |     mkdir -p "${root}/etc/paths.d"
135 |     cat >"${root}/etc/paths.d/pkg_comp" <<EOF
136 | /usr/local/bin
137 | /usr/local/sbin
138 | /opt/pkg/bin
139 | /opt/pkg/sbin
140 | EOF
141 | 
142 |     for dir in sandboxctl pkg_comp; do
143 |         mkdir -p "${root}/usr/local/share/examples/${dir}"
144 |         mv "${root}/usr/local/etc/${dir}"/* \
145 |             "${root}/usr/local/share/examples/${dir}"
146 |     done
147 | 
148 |     # TODO(jmmv): We should ship this ourselves as part of the pkg_comp release
149 |     # and use the file here and in pkgsrc's pkg_comp-cron.  Maybe even change
150 |     # the semantics of the AUTO_PACKAGES variable to point to a file and handle
151 |     # this internally for better file parsing.
152 |     cat >"${root}/usr/local/share/examples/pkg_comp/list.txt" <<EOF
153 | # Packages to build automatically.
154 | #
155 | # List one package per line, using bare names or the category/name syntax.
156 | 
157 | #pkgtools/pkgin
158 | #tmux
159 | EOF
160 | 
161 |     mkdir -p "${root}/var/pkg_comp"
162 |     mkdir -p "${root}/var/pkg_comp/log"
163 | 
164 |     local f="${root}/usr/local/share/examples/pkg_comp/default.conf"
165 |     edit_config "${f}" FETCH_VCS git
166 |     edit_config "${f}" PKGSRCDIR /var/pkg_comp/pkgsrc
167 |     edit_config "${f}" DISTDIR /var/pkg_comp/distfiles
168 |     edit_config "${f}" PACKAGES /var/pkg_comp/packages
169 |     edit_config "${f}" PBULK_LOG /var/pkg_comp/log/pbulk
170 |     edit_config "${f}" PBULK_PACKAGES /var/pkg_comp/pbulk-packages
171 |     edit_config "${f}" LOCALBASE /opt/pkg
172 |     edit_config "${f}" PKG_DBDIR /opt/pkg/libdata/pkgdb
173 |     edit_config "${f}" SYSCONFDIR /opt/pkg/etc
174 |     edit_config "${f}" VARBASE /opt/pkg/var
175 |     edit_config "${f}" \
176 |         AUTO_PACKAGES "\$(grep -v ^# /usr/local/etc/pkg_comp/list.txt)"
177 | 
178 |     local f="${root}/usr/local/share/examples/pkg_comp/sandbox.conf"
179 |     edit_config "${f}" SANDBOX_ROOT /var/pkg_comp/sandbox
180 | 
181 |     mkdir -p "${root}/usr/local/libexec/pkg_comp"
182 |     cat >"${root}/usr/local/libexec/pkg_comp/uninstall.sh" <<EOF
183 | #! /bin/sh
184 | 
185 | tempfile="\$(mktemp "\${TMPDIR:-/tmp}/pkg_comp.XXXXXX")"
186 | if crontab -u root -l >>"\${tempfile}"; then
187 |     if grep /usr/local/sbin/pkg_comp "\${tempfile}" >/dev/null; then
188 |         tempfile2="\$(mktemp "\${TMPDIR:-/tmp}/pkg_comp.XXXXXX")"
189 |         grep -v /usr/local/sbin/pkg_comp "\${tempfile}" >>"\${tempfile2}"
190 |         crontab -u root "\${tempfile2}"
191 |         rm -f "\${tempfile2}"
192 |     fi
193 | fi
194 | rm -f "\${tempfile}"
195 | 
196 | cd /
197 | for f in \$(tail -r /usr/local/share/pkg_comp/manifest); do
198 |     if [ ! -d "\${f}" ]; then
199 |         rm "\${f}"
200 |     else
201 |         rmdir "\${f}"
202 |     fi
203 | done
204 | EOF
205 |     chmod +x "${root}/usr/local/libexec/pkg_comp/uninstall.sh"
206 | 
207 |     ( cd "${root}" && find . >"${root}/usr/local/share/pkg_comp/manifest" )
208 | }
209 | 
210 | 
211 | # Program's entry point.
212 | main() {
213 |     [ "$(uname -s)" = Darwin ] || err "This script is for macOS only"
214 |     [ -x "${DirName}/bootstrap.sh" ] || err "bootstrap.sh not found; make" \
215 |         "sure to run this from a cloned repository"
216 |     [ "$(id -u)" -eq 0 ] || err "Must be run as root"
217 | 
218 |     local tempdir
219 |     tempdir="$(mktemp -d "${TMPDIR:-/tmp}/${ProgName}.XXXXXX" 2>/dev/null)" \
220 |         || err "Failed to create temporary directory"
221 |     trap "rm -rf '${tempdir}'" EXIT
222 | 
223 |     export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig  # For OSXFUSE.
224 | 
225 |     # Bootstrap under a clean prefix to generate a manifest of all the files
226 |     # required by our installation.
227 |     info "Doing temporary bootstrap to generate file manifest"
228 |     "${DirName}/bootstrap.sh" "${@}" "${tempdir}/prefix" \
229 |         || err "Bootstrap failed"
230 |     ( cd "${tempdir}/prefix" && find . \! -type d ) >"${tempdir}/manifest"
231 | 
232 |     # Bootstrap under the final location.
233 |     info "Doing real bootstrap under prefix"
234 |     "${DirName}/bootstrap.sh" "${@}" /usr/local || err "Bootstrap failed"
235 | 
236 |     info "Generating package root"
237 |     mkdir -p "${tempdir}/root/usr/local"
238 |     tar -cp -C /usr/local -f - $(cat "${tempdir}/manifest") | \
239 |         tar -xp -C "${tempdir}/root/usr/local" -f -
240 |     configure_root "${tempdir}/root"
241 | 
242 |     info "Removing pkg_comp installation from /usr/local"
243 |     ( cd /usr/local && rm $(cat "${tempdir}/manifest") )
244 | 
245 |     local version="$(grep '^Changes in version' \
246 |         "${tempdir}/prefix/share/doc/pkg_comp/NEWS" \
247 |         | head -n 1 | awk '{print $4}')"
248 |     local revision="$(date +%Y%m%d)"
249 |     local pkgversion="${version}-${revision}"
250 |     local pkgfile="pkg_comp-${pkgversion}-macos.pkg"
251 | 
252 |     info "Building package ${pkgfile}"
253 |     write_scripts "${tempdir}/scripts"
254 |     pkgbuild \
255 |         --identifier com.github.jmmv.pkg_comp \
256 |         --root "${tempdir}/root" \
257 |         --scripts "${tempdir}/scripts" \
258 |         --version "${pkgversion}" \
259 |         "${pkgfile}"
260 | }
261 | 
262 | 
263 | main "${@}"
264 | 


--------------------------------------------------------------------------------
/git_test.sh:
--------------------------------------------------------------------------------
  1 | # Copyright 2017 Google Inc.
  2 | # All rights reserved.
  3 | #
  4 | # Redistribution and use in source and binary forms, with or without
  5 | # modification, are permitted provided that the following conditions are
  6 | # met:
  7 | #
  8 | # * Redistributions of source code must retain the above copyright
  9 | #   notice, this list of conditions and the following disclaimer.
 10 | # * Redistributions in binary form must reproduce the above copyright
 11 | #   notice, this list of conditions and the following disclaimer in the
 12 | #   documentation and/or other materials provided with the distribution.
 13 | # * Neither the name of Google Inc. nor the names of its contributors
 14 | #   may be used to endorse or promote products derived from this software
 15 | #   without specific prior written permission.
 16 | #
 17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 18 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 19 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 20 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 21 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 22 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 23 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 24 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 25 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 26 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 28 | 
 29 | shtk_import unittest
 30 | : ${PKG_COMP_SHTK_MODULESDIR:="__PKG_COMP_SHTK_MODULESDIR__"}
 31 | SHTK_MODULESPATH="${PKG_COMP_SHTK_MODULESDIR}" shtk_import git
 32 | 
 33 | 
 34 | # Creates a local Git repository with some files.
 35 | #
 36 | # \param dir Path to the repository to create.
 37 | # \param branch Name of the branch to create.  Avoid "master" to ensure our
 38 | #     functions work with the non-default branch.
 39 | init_git_repository() {
 40 |     local dir="${1}"; shift
 41 |     local branch="${1}"; shift
 42 | 
 43 |     # Git can, in theory, automatically set some default values for these.
 44 |     # Unfortunately, such autoconfiguration can fail if e.g. the domain name
 45 |     # for the machine is missing.  Be explicit.
 46 |     assert_command -o ignore -e ignore git config --global user.email \
 47 |         test@example.com
 48 |     assert_command -o ignore -e ignore git config --global user.name Test
 49 | 
 50 |     assert_command -o ignore -e ignore git init --bare "${dir}"
 51 | 
 52 |     mkdir work
 53 |     cd work
 54 |     assert_command -o ignore -e ignore git init
 55 |     assert_command -o ignore -e ignore git checkout -b "${branch}"
 56 |     echo "first revision" >the-file
 57 |     assert_command -o ignore -e ignore git add the-file
 58 |     assert_command -o ignore -e ignore git commit -a -m "First revision"
 59 |     assert_command -o ignore -e ignore git push -u "file://${dir}" "${branch}"
 60 |     cd -
 61 |     rm -rf work
 62 | }
 63 | 
 64 | 
 65 | # Commits and pushes pending changes in a working copy.
 66 | #
 67 | # \param dir Path to the git working copy.
 68 | commit_and_push() {
 69 |     local dir="${1}"; shift
 70 | 
 71 |     (
 72 |         cd "${dir}"
 73 |         git commit -a -m "Changes." || exit 1
 74 |         git push || exit 1
 75 |     ) || fail "Failed to modify repository"
 76 | }
 77 | 
 78 | 
 79 | shtk_unittest_add_fixture fetch
 80 | fetch_fixture() {
 81 |     setup() {
 82 |         REPOSITORY_DIR="$(pwd)/repository"
 83 |         REPOSITORY_URL="file://${REPOSITORY_DIR}"
 84 |         init_git_repository "${REPOSITORY_DIR}" trunk
 85 |     }
 86 | 
 87 | 
 88 |     teardown() {
 89 |         rm -rf "${REPOSITORY_DIR}"
 90 |     }
 91 | 
 92 | 
 93 |     shtk_unittest_add_test ok
 94 |     ok_test() {
 95 |         expect_command -o ignore -e ignore \
 96 |             git_fetch "${REPOSITORY_URL}" trunk clone1
 97 |         grep "first revision" clone1/the-file >/dev/null \
 98 |             || fail "Unexpected version found"
 99 | 
100 |         cp -r clone1 clone2
101 |         echo "second revision" >clone2/the-file
102 |         commit_and_push clone2
103 |         rm -rf clone2
104 | 
105 |         grep "first revision" clone1/the-file >/dev/null \
106 |             || fail "Unexpected version found"
107 |         expect_command -o ignore -e ignore \
108 |             git_fetch "${REPOSITORY_URL}" trunk clone1
109 |         grep "second revision" clone1/the-file >/dev/null \
110 |             || fail "Unexpected version found"
111 |     }
112 | }
113 | 
114 | 
115 | shtk_unittest_add_fixture checkout
116 | checkout_fixture() {
117 |     setup() {
118 |         REPOSITORY_DIR="$(pwd)/repository"
119 |         REPOSITORY_URL="file://${REPOSITORY_DIR}"
120 |     }
121 | 
122 | 
123 |     teardown() {
124 |         rm -rf "${REPOSITORY_DIR}"
125 |     }
126 | 
127 | 
128 |     shtk_unittest_add_test ok
129 |     ok_test() {
130 |         init_git_repository "${REPOSITORY_DIR}" trunk
131 |         expect_command -o ignore -e ignore \
132 |             git_clone "${REPOSITORY_URL}" trunk clone
133 |         grep "first revision" clone/the-file >/dev/null \
134 |             || fail "Unexpected version found"
135 |     }
136 | 
137 | 
138 |     shtk_unittest_add_test already_exists
139 |     already_exists_test() {
140 |         mkdir -p missing-dir
141 |         expect_command -s exit:1 \
142 |             -e match:"Cannot clone into .*missing-dir.* exists" \
143 |             git_clone "${REPOSITORY_URL}" trunk missing-dir
144 |     }
145 | 
146 | 
147 |     shtk_unittest_add_test git_fails
148 |     git_fails_test() {
149 |         init_git_repository "${REPOSITORY_DIR}" trunk
150 |         expect_command -s exit:1 -e match:"Git clone failed" \
151 |             git_clone "${REPOSITORY_URL}" non-existent dir
152 |         [ ! -e dir ] || fail "Clone directory created and left behind"
153 |     }
154 | }
155 | 
156 | 
157 | shtk_unittest_add_fixture update
158 | update_fixture() {
159 |     setup() {
160 |         REPOSITORY_DIR="$(pwd)/repository"
161 |         REPOSITORY_URL="file://${REPOSITORY_DIR}"
162 |     }
163 | 
164 | 
165 |     teardown() {
166 |         rm -rf "${REPOSITORY_DIR}"
167 |     }
168 | 
169 | 
170 |     shtk_unittest_add_test ok
171 |     ok_test() {
172 |         init_git_repository "${REPOSITORY_DIR}" trunk
173 | 
174 |         assert_command -o ignore -e ignore \
175 |             git clone -b trunk "${REPOSITORY_URL}" first
176 |         assert_command -o ignore -e ignore \
177 |             git clone -b trunk "${REPOSITORY_URL}" copy
178 | 
179 |         expect_command -o ignore -e ignore \
180 |             git_update "${REPOSITORY_URL}" trunk first
181 |         grep "first revision" first/the-file >/dev/null \
182 |             || fail "Unexpected version found"
183 | 
184 |         echo "second revision" >copy/the-file
185 |         commit_and_push copy
186 | 
187 |         expect_command -o ignore -e ignore \
188 |             git_update "${REPOSITORY_URL}" trunk first
189 |         grep "second revision" first/the-file >/dev/null \
190 |             || fail "Unexpected version found"
191 |     }
192 | 
193 | 
194 |     shtk_unittest_add_test stash_changes__ok
195 |     stash_changes__ok_test() {
196 |         init_git_repository "${REPOSITORY_DIR}" trunk
197 | 
198 |         # Make the-file contain multiple lines so that we can perform
199 |         # independent edits in two places.
200 |         assert_command -o ignore -e ignore \
201 |             git clone -b trunk "${REPOSITORY_URL}" first
202 |         seq 10 >first/the-file
203 |         commit_and_push first
204 | 
205 |         # Update a line in the-file and push it to the repository.
206 |         assert_command -o ignore -e ignore \
207 |             git clone -b trunk "${REPOSITORY_URL}" copy
208 |         sed s,^9$,90, copy/the-file >copy/the-file.new
209 |         mv copy/the-file.new copy/the-file
210 |         commit_and_push copy
211 | 
212 |         # Modify a different line in the-file that is far enough from the other
213 |         # line we modified so that the two edits don't conflict, and leave the
214 |         # changes uncommitted.
215 |         sed s,^2$,20, first/the-file >first/the-file.new
216 |         mv first/the-file.new first/the-file
217 |         expect_command -o ignore -e ignore \
218 |             git_update "${REPOSITORY_URL}" trunk first
219 | 
220 |         # Sanity-check that our edits remain.
221 |         cat >expout <<EOF
222 | 1
223 | 20
224 | 3
225 | 4
226 | 5
227 | 6
228 | 7
229 | 8
230 | 90
231 | 10
232 | EOF
233 |         expect_command -o file:expout cat first/the-file
234 | 
235 |         # Sanity-check that our uncommitted changes are still uncommitted.
236 |         (
237 |             cd first
238 |             git status --porcelain >../status
239 |         ) || fail "git status failed"
240 |         cat >expout <<EOF
241 |  M the-file
242 | EOF
243 |         expect_command -o file:expout cat status
244 |     }
245 | 
246 | 
247 |     shtk_unittest_add_test stash_changes__conflict
248 |     stash_changes__conflict_test() {
249 |         init_git_repository "${REPOSITORY_DIR}" trunk
250 | 
251 |         # Make the-file contain multiple lines so that we can perform
252 |         # independent edits in two places.
253 |         assert_command -o ignore -e ignore \
254 |             git clone -b trunk "${REPOSITORY_URL}" first
255 |         seq 10 >first/the-file
256 |         commit_and_push first
257 | 
258 |         # Update a line in the-file and push it to the repository.
259 |         assert_command -o ignore -e ignore \
260 |             git clone -b trunk "${REPOSITORY_URL}" copy
261 |         sed s,^9$,90, copy/the-file >copy/the-file.new
262 |         mv copy/the-file.new copy/the-file
263 |         commit_and_push copy
264 | 
265 |         # Modify a different line in the-file that is close enough to the other
266 |         # line we modified so that the two edits conflict, and leave the changes
267 |         # uncommitted.
268 |         sed s,^8$,80, first/the-file >first/the-file.new
269 |         mv first/the-file.new first/the-file
270 |         expect_command -s exit:1 -o ignore -e ignore \
271 |             git_update "${REPOSITORY_URL}" trunk first
272 | 
273 |         # Check that the file is left behind with conflict markers.
274 |         grep '<<<<' first/the-file || fail "No conflict markers found"
275 |     }
276 | 
277 | 
278 |     shtk_unittest_add_test switch_branch
279 |     switch_branch_test() {
280 |         init_git_repository "${REPOSITORY_DIR}" trunk
281 | 
282 |         assert_command -o ignore -e ignore \
283 |             git clone -b trunk "${REPOSITORY_URL}" first
284 |         (
285 |             cd first
286 |             git checkout -b other-trunk
287 |             echo "alternate revision" >the-file
288 |             git commit -a -m "Other branch" || exit 1
289 |             git push "${REPOSITORY_URL}" other-trunk || exit 1
290 | 
291 |             git checkout trunk || exit 1
292 |         ) || fail "Failed to create new branch"
293 | 
294 |         grep "first revision" first/the-file >/dev/null \
295 |             || fail "Unexpected version found"
296 |         expect_command -o ignore -e ignore \
297 |             git_update "${REPOSITORY_URL}" other-trunk first
298 |         grep "alternate revision" first/the-file >/dev/null \
299 |             || fail "Unexpected version found"
300 |     }
301 | 
302 | 
303 |     shtk_unittest_add_test does_not_exist
304 |     does_not_exist_test() {
305 |         expect_command -s exit:1 -e match:"Cannot update src; .*not exist" \
306 |             git_update "${REPOSITORY_URL}" trunk src
307 |     }
308 | 
309 | 
310 |     shtk_unittest_add_test git_fails
311 |     git_fails_test() {
312 |         init_git_repository "${REPOSITORY_DIR}" trunk
313 |         assert_command -o ignore -e ignore \
314 |             git clone -b trunk "${REPOSITORY_URL}" work
315 |         expect_command -s exit:1 -e match:"Git fetch failed" \
316 |             git_update "${REPOSITORY_URL}" bad-branch work
317 |     }
318 | }
319 | 


--------------------------------------------------------------------------------
/pkg_comp.8.in:
--------------------------------------------------------------------------------
  1 | .\" Copyright 2013 Google Inc.
  2 | .\" All rights reserved.
  3 | .\"
  4 | .\" Redistribution and use in source and binary forms, with or without
  5 | .\" modification, are permitted provided that the following conditions are
  6 | .\" met:
  7 | .\"
  8 | .\" * Redistributions of source code must retain the above copyright
  9 | .\"   notice, this list of conditions and the following disclaimer.
 10 | .\" * Redistributions in binary form must reproduce the above copyright
 11 | .\"   notice, this list of conditions and the following disclaimer in the
 12 | .\"   documentation and/or other materials provided with the distribution.
 13 | .\" * Neither the name of Google Inc. nor the names of its contributors
 14 | .\"   may be used to endorse or promote products derived from this software
 15 | .\"   without specific prior written permission.
 16 | .\"
 17 | .\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 18 | .\" "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 19 | .\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 20 | .\" A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 21 | .\" OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 22 | .\" SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 23 | .\" LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 24 | .\" DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 25 | .\" THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 26 | .\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 27 | .\" OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 28 | .Dd August 30, 2018
 29 | .Dt PKG_COMP 8
 30 | .Os
 31 | .Sh NAME
 32 | .Nm pkg_comp
 33 | .Nd builds pkgsrc packages in a chroot environment
 34 | .Sh SYNOPSIS
 35 | .Nm
 36 | .Op Fl c Ar config_name
 37 | .Op Fl o Ar variable=value
 38 | auto
 39 | .Op Fl f
 40 | .Op pkg1 Ar .. pkgN
 41 | .Nm
 42 | .Op Fl c Ar config_name
 43 | .Op Fl o Ar variable=value
 44 | bootstrap
 45 | .Nm
 46 | .Op Fl c Ar config_name
 47 | .Op Fl o Ar variable=value
 48 | build
 49 | .Ar pkg1
 50 | .Op Ar .. pkgN
 51 | .Nm
 52 | .Op Fl c Ar config_name
 53 | .Op Fl o Ar variable=value
 54 | config
 55 | .Nm
 56 | .Op Fl c Ar config_name
 57 | .Op Fl o Ar variable=value
 58 | fetch
 59 | .Nm
 60 | .Op Fl c Ar config_name
 61 | .Op Fl o Ar variable=value
 62 | sandbox-<command>
 63 | .Op Ar arg1 .. argN
 64 | .Sh DESCRIPTION
 65 | .Nm
 66 | is a utility to build pkgsrc packages in a fully automated and self-contained
 67 | manner.
 68 | The process is made unattended by relying on a configuration file that
 69 | prespecifies the location of all build components, the environment in which to
 70 | build the packages (aka the layout of the chroot sandbox), and the desired goals
 71 | of the build.
 72 | If you want to plug
 73 | .Nm
 74 | into
 75 | .Xr cron 8 ,
 76 | please take a look to
 77 | .Xr pkg_comp4cron 8
 78 | now.
 79 | .Pp
 80 | .Nm
 81 | can be seen as a wrapper over
 82 | .Xr cvs 1
 83 | and
 84 | .Xr git 1 ,
 85 | the pbulk infrastructure provided by pkgsrc, and
 86 | .Xr sandboxctl 8 .
 87 | .Nm
 88 | provides the following additional features:
 89 | .Bl -bullet
 90 | .It
 91 | Automatically fetch or update the pkgsrc tree before performing a build.
 92 | .It
 93 | Preconfigure the way binary packages are built (file system layout, package
 94 | options, etc.) by storing all relevant details in a configuration file.
 95 | .It
 96 | Preconfigure a clean environment (chroot sandbox) in which the packages are
 97 | built to avoid system-wide side-effects and contamination.
 98 | .It
 99 | Trivially set up periodic pkgsrc rebuilds by adding a single line to your
100 | .Xr crontab 5 .
101 | .El
102 | .Pp
103 | As you can see in the
104 | .Sx SYNOPSIS
105 | section,
106 | .Nm
107 | provides a subcommand-interface: the tool has several modes of operation,
108 | and the particular mode to use for a given run is selected by the first
109 | non-option argument in the command line.
110 | .Pp
111 | The following options apply to all commands:
112 | .Bl -tag -width XXXX
113 | .It Fl c Ar config_name
114 | Specifies the configuration file to use.
115 | The format of the file is described in
116 | .Xr pkg_comp.conf 5 .
117 | .Pp
118 | If
119 | .Ar config_name
120 | includes any directory separator (aka, one or more slashes) or the
121 | .Sq .conf
122 | suffix, then this specifies the path of the configuration file to load.
123 | .Pp
124 | If
125 | .Ar config_name
126 | is a plain name without any directory components nor extension, then this
127 | specifies the name of the configuration.
128 | In this case,
129 | .Nm
130 | will use
131 | .Pa __PKG_COMP_ETCDIR__/<config_name>.conf ,
132 | which must exist; otherwise, the tool exits with an error.
133 | .It Fl o Ar variable=value
134 | Applies an override to the loaded configuration.
135 | .Pp
136 | The
137 | .Ar variable
138 | part of the argument must be any of the recognized configuration variables
139 | described in
140 | .Xr pkg_comp.conf 5 .
141 | The
142 | .Ar value ,
143 | if not empty, specifies the value to set the configuration variable to.
144 | If
145 | .Ar value
146 | is empty, then the configuration variable is unset.
147 | .El
148 | .Ss The auto command
149 | The auto command provides the main functionality of
150 | .Nm
151 | as it orchestrates all other subcommands in an unattended manner.
152 | This command allows you to trivially rebuild all the packages you are interested
153 | in at will, even as a periodic
154 | .Xr cron 8
155 | job; see
156 | .Xr pkg_comp4cron 8
157 | if you are interested in doing this.
158 | .Pp
159 | The auto command performs the following steps:
160 | .Bl -enum
161 | .It
162 | If
163 | .Va UPDATE_SOURCES
164 | is true, the fetch command is invoked first.
165 | See
166 | .Sx The fetch command
167 | for more details.
168 | .It
169 | If the sandbox does not exist yet, creates the sandbox by invoking the
170 | sandbox-create command.
171 | .It
172 | Bootstraps pkgsrc and pbulk within the sandbox by invoking the bootstrap
173 | command.
174 | See
175 | .Sx The bootstrap command
176 | for more details.
177 | .It
178 | Builds the set of packages given either in the
179 | .Va AUTO_PACKAGES
180 | configuration variable or on the command line by passing those to the
181 | build command.
182 | See
183 | .Sx The build command
184 | for more details.
185 | .It
186 | If the sandbox was created by this command, destroys the sandbox by invoking
187 | the sandbox-destroy command.
188 | .El
189 | .Pp
190 | The following options apply only to the auto command:
191 | .Bl -tag -width XXXX
192 | .It Fl f
193 | Enables fast mode, which skips updating the source tree.
194 | This is a shorthand for these generic flags:
195 | .Fl o Ar UPDATE_SOURCES=false .
196 | .El
197 | .Ss The bootstrap command
198 | The bootstrap command prepares the pkgsrc infrastructure within the sandbox to
199 | be ready to build packages.
200 | This involves two steps:
201 | .Bl -enum
202 | .It
203 | Bootstraps pkgsrc and installs pbulk under the sandboxes'
204 | .Pa /pkg_comp/pbulk
205 | tree.
206 | This installation is used internally by
207 | .Nm
208 | thus is not configurable.
209 | .It
210 | Bootstraps pkgsrc in the location configured by the user via the
211 | .Va LOCALBASE
212 | et. al. configuration variables.
213 | .El
214 | .Pp
215 | On completion, the packages directory configured by the
216 | .Va PACKAGES
217 | configuration variable will contain the binary bootstrap kit and any
218 | bootstrap-related binary packages for the target environment, and
219 | .Va PBULK_PACKAGES
220 | will contain the same artifacts but for the pbulk environment used internally by
221 | .Nm .
222 | .Pp
223 | Any existing binaries within the
224 | .Va PBULK_PACKAGES
225 | tree will be reused on subsequent bootstrap operations if found.
226 | If you want to start builds afresh (and you should if you experience strange
227 | build failures), you will have to remove the previous binary packages.
228 | .Ss The build command
229 | The build command triggers the build of a set of packages within the
230 | sandbox via pbulk.
231 | The sandbox must have already been created with the sandbox-create command, but
232 | it need not have been bootstrapped yet; create takes care of bootstrapping if
233 | necessary.
234 | .Pp
235 | The packages to build can be provided either as bare names, such as
236 | .Sq tmux ,
237 | or as category/package pairs, such as
238 | .Sq misc/tmux .
239 | .Pp
240 | On completion, the binary packages are left in the directory specified by the
241 | .Va PACKAGES
242 | configuration variable, which can later be fed to other tools such as
243 | .Xr pkg_add 8
244 | or
245 | .Xr pkgin 8
246 | to install the resulting binary packages.
247 | .Pp
248 | Note that, due to the way pbulk works, this command will build the packages
249 | provided on the command line
250 | .Em and all other packages previously built within the sandbox .
251 | This is necessary to keep fully-populated
252 | .Xr pkg_summary 5
253 | databases.
254 | However, because pbulk reuses existing binary packages and only rebuilds updated
255 | packages, any packages not given on the command line should "build" quickly.
256 | .Pp
257 | In case of build problems, detailed logs for each package that failed to build
258 | will be located in the directory pointed at by
259 | .Va PBULK_LOG .
260 | Of special interest is the
261 | .Pa SUMMARY
262 | subdirectory, which contains summary reports of the whole operation in various
263 | different formats.
264 | Examples are:
265 | .Pa report.html
266 | and
267 | .Pa report.txt ,
268 | both of which summarize how many packages were built and include details of all
269 | that failed.
270 | .Ss The config command
271 | The config command dumps the loaded configuration to the standard output.
272 | The format of the output is not a script, so it cannot be fed back into
273 | .Nm .
274 | The purpose of this command is to aid in debugging the configuration of the
275 | tool before performing any builds, particularly when the configuration
276 | files use shell logic to determine the value of any variables.
277 | .Pp
278 | The output contains two separate "paragraphs": the first lists all options
279 | specific to
280 | .Nm ,
281 | including those given on the command line and in
282 | .Xr pkg_comp.conf 5 ;
283 | and the second lists all options passed to
284 | .Xr sandboxctl 8
285 | as specified in the configuration file pointed at by
286 | .Va SANDBOX_CONFFILE .
287 | .Ss The fetch command
288 | The fetch command downloads or updates the pkgsrc tree.
289 | .Pp
290 | If the tree does not exist yet in the location specified by
291 | .Va PKGSRCDIR ,
292 | this performs an initial checkout using the version control system of choice;
293 | otherwise, this updates the local copy, preserving any changes that may exist.
294 | Note that local changes that conflict with remote changes will cause the fetch
295 | operation to fail.
296 | .Pp
297 | The
298 | .Va CVS_ROOT ,
299 | .Va CVS_TAG ,
300 | .Va FETCH_VCS ,
301 | .Va GIT_BRANCH
302 | and
303 | .Va GIT_URL
304 | variables are used to determine where to get the sources from and whether a
305 | particular branch is desired.
306 | .Ss Sandbox manipulation commands
307 | The
308 | .Sq sandbox-<command>
309 | family of commands are simple wrappers around
310 | .Xr sandboxctl 8
311 | using the sandbox configuration specified in the
312 | .Va SANDBOX_CONFFILE
313 | configuration variable.
314 | .Sh FILES
315 | .Bl -tag -width XXXX
316 | .It Pa __PKG_COMP_ETCDIR__/
317 | Directory containing all system-wide configuration files.
318 | .It Pa __PKG_COMP_ETCDIR__/default.conf
319 | Default configuration file to load if the
320 | .Fl c
321 | flag is not provided.
322 | .It Pa __PKG_COMP_ETCDIR__/extra.mk.conf
323 | Sample contents for possible free-form
324 | .Xr mk.conf 5
325 | extensions.
326 | This file is explicitly referenced from
327 | .Pa default.conf
328 | by the
329 | .Va EXTRA_MKCONF
330 | configuration variable.
331 | .It Pa __PKG_COMP_ETCDIR__/sandbox.conf
332 | Sample contents for the
333 | .Xr sandboxctl 8
334 | configuration file to use in the build of packages.
335 | This file is explicitly referenced from
336 | .Pa default.conf
337 | by the
338 | .Va SANDBOX_CONFFILE
339 | configuration variable.
340 | .El
341 | .Sh EXAMPLES
342 | The following examples assume that you have already configured
343 | .Xr pkg_comp.conf 5
344 | and its corresponding
345 | .Xr sandbox.conf 5
346 | file.
347 | .Pp
348 | To build a collection of packages from scratch, on a system in which you may not
349 | yet have pkgsrc nor an existent sandbox:
350 | .Bd -literal -offset indent
351 | # pkg_comp auto bash emacs sudo tmux
352 | .Ed
353 | .Pp
354 | To manually control the lifecycle of the sandbox, the updates to the pkgsrc
355 | tree, and to build the same set of packages, you would do:
356 | .Bd -literal -offset indent
357 | # pkg_comp sandbox-create
358 | # pkg_comp fetch
359 | # pkg_comp build bash emacs sudo tmux
360 | # pkg_comp sandbox-destroy
361 | .Ed
362 | .Pp
363 | After the commands above, and under the default configuration, you will find
364 | the resulting binary packages under
365 | .Pa /usr/pkgsrc/packages/All .
366 | All you need to do to use them via
367 | .Xr pkg_add 8
368 | is:
369 | .Bd -literal -offset indent
370 | # PKG_PATH=file:///usr/pkgsrc/packages/All; export PKG_PATH
371 | # pkg_add sudo
372 | .Ed
373 | .Sh SEE ALSO
374 | .Xr cvs 1 ,
375 | .Xr pkg_comp.conf 5 ,
376 | .Xr hier 7 ,
377 | .Xr pkg_comp4cron 8 ,
378 | .Xr sandboxctl 8
379 | .Sh HISTORY
380 | .Nm
381 | 1.x first appeared in pkgsrc on September 6th, 2002 as the
382 | .Pa pkgtools/pkg_comp
383 | package and was later moved to
384 | .Pa pkgtools/pkg_comp1
385 | on February 12th, 2017, to make room for the 2.x series.
386 | The old code was moved aside, instead of just being replaced, because the
387 | 1.x series are completely different to the 2.x series and the configuration
388 | cannot be trivially migrated.
389 | .Pp
390 | .Nm
391 | 2.x, documented here, first appeared on February 17th, 2017 as a full rewrite
392 | of the original tool.
393 | The rewrite had been in the works for years, seeing multiple iterations.
394 | The first was sometime in 2006, when I attempted to rewrite
395 | .Nm
396 | in the language I had just learned: Haskell.
397 | The second was sometime in 2010 or so, when I tried to rewrite the tool
398 | in Python soon after I joined Google as an SRE.
399 | The third, which is what you witness here together with the separate
400 | .Xr sandboxctl 8
401 | tool, started in 2013 but was put on hold pretty much until 2017 due to me
402 | spending more time with family.
403 | I was finally able to make the time to complete this rewrite by shifting my
404 | schedule to start the day at 5am.
405 | .Pp
406 | The main reasons for the 2.x rewrite, which involve a complete departure from
407 | the codebase of 1.x and its configuration interface, were the desire to make
408 | .Nm
409 | work across platforms, to integrate with pkgsrc's bootstrap and pbulk
410 | technologies, and to make it testable to ensure its reliability.
411 | .Pp
412 | See
413 | .Pa __PKG_COMP_DOCDIR__/NEWS
414 | for more details on the changes across releases.
415 | .Sh AUTHORS
416 | The
417 | .Nm
418 | utility was developed by
419 | .An Julio Merino
420 | .Aq jmmv@google.com .
421 | .Sh BUGS
422 | The following are known limitations in the current
423 | .Nm
424 | implementation.
425 | .Bl -bullet
426 | .It
427 | .Nm
428 | unconditionally configures a pkgsrc bootstrap kit for the target location given
429 | by the user in
430 | .Va LOCALBASE .
431 | This is intentional for simplicity and consistency across platforms but means
432 | that the
433 | .Xr pkg_add 8
434 | family of tools and
435 | .Xr make 1
436 | provided by the base system are ignored.
437 | .It
438 | Even though pkgsrc supports unprivileged builds,
439 | .Nm
440 | currently requires root.
441 | The reason is that
442 | .Nm
443 | relies on a sandbox created by
444 | .Xr sandboxctl 8
445 | to build packages, and entering a sandbox with
446 | .Xr chroot 8
447 | requires root privileges.
448 | It would be nice to lift this restriction so that users could rebuild personal
449 | sets of binary packages under their home directory.
450 | One option would be to make the use of a sandbox optional, with the
451 | understanding that not using a sandbox would leave any current set of installed
452 | packages non-functional during the build.
453 | .It
454 | This rewrite of
455 | .Nm
456 | still lacks some interesting features from the 1.x line.
457 | For example, support for
458 | .Xr kver 3
459 | or the vulnerabilities database.
460 | .El
461 | 


--------------------------------------------------------------------------------
/pkg_comp_test.sh:
--------------------------------------------------------------------------------
  1 | #! __ATF_SH__
  2 | # Copyright 2013 Google Inc.
  3 | # All rights reserved.
  4 | #
  5 | # Redistribution and use in source and binary forms, with or without
  6 | # modification, are permitted provided that the following conditions are
  7 | # met:
  8 | #
  9 | # * Redistributions of source code must retain the above copyright
 10 | #   notice, this list of conditions and the following disclaimer.
 11 | # * Redistributions in binary form must reproduce the above copyright
 12 | #   notice, this list of conditions and the following disclaimer in the
 13 | #   documentation and/or other materials provided with the distribution.
 14 | # * Neither the name of Google Inc. nor the names of its contributors
 15 | #   may be used to endorse or promote products derived from this software
 16 | #   without specific prior written permission.
 17 | #
 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 19 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 20 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 21 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 22 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 23 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 24 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 25 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 26 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 27 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 28 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 29 | 
 30 | # \file pkg_comp_test.sh
 31 | # Tests for the pkg_comp.sh script.
 32 | #
 33 | # The tests in this script are intended to be fast to run but, for the most
 34 | # part, use only mocks to verify that the main program logic works.
 35 | 
 36 | 
 37 | # Paths to installed files.
 38 | #
 39 | # Can be overriden for test purposes only.
 40 | : ${PKG_COMP_ETCDIR="__PKG_COMP_ETCDIR__"}
 41 | : ${PKG_COMP_SHAREDIR="__PKG_COMP_SHAREDIR__"}
 42 | 
 43 | 
 44 | # Path to a local cvsroot for testing purposes.
 45 | MOCK_CVSROOT=":local:$(pwd)/cvsroot"
 46 | 
 47 | 
 48 | # Creates a fake program that records its invocations for later processing.
 49 | #
 50 | # The fake program, when invoked, will append its arguments to a commands.log
 51 | # file in the test case's work directory.
 52 | #
 53 | # \param binary The path to the program to create.
 54 | # \param delegate If set to 'yes', execute the real program afterwards.
 55 | create_mock_binary() {
 56 |     local binary="${1}"; shift
 57 |     local delegate=no
 58 |     [ ${#} -eq 0 ] || { delegate="${1}"; shift; }
 59 | 
 60 |     cat >"${binary}" <<EOF
 61 | #! /bin/sh
 62 | 
 63 | logfile="${HOME}/commands.log"
 64 | echo "Command: \${0##*/}" >>"\${logfile}"
 65 | echo "Directory: \$(pwd)" >>"\${logfile}"
 66 | for arg in "\${@}"; do
 67 |     echo "Arg: \${arg}" >>"\${logfile}"
 68 | done
 69 |     echo >>"\${logfile}"
 70 | EOF
 71 | 
 72 |     if [ "${delegate}" = yes ]; then
 73 |         cat >>"${binary}" <<EOF
 74 | PATH="${PATH}"
 75 | exec "\${0##*/}" "\${@}"
 76 | EOF
 77 |     fi
 78 | 
 79 |     chmod +x "${binary}"
 80 | }
 81 | 
 82 | 
 83 | # Creates a fake CVS repository with a pkgsrc module.
 84 | #
 85 | # \param repository Path to the repository to create.
 86 | create_mock_cvsroot() {
 87 |     local repository="${1}"; shift
 88 | 
 89 |     atf_check -o ignore -e ignore cvs -d "${repository}" init
 90 | 
 91 |     mkdir pkgsrc
 92 |     cd pkgsrc
 93 |     create_mock_binary build.sh
 94 |     echo "first revision" >file-in-pkgsrc
 95 |     cvs -d "${repository}" import -m "Import." pkgsrc VENDOR_TAG release_tag
 96 |     cd -
 97 |     rm -rf pkgsrc
 98 | }
 99 | 
100 | 
101 | # Creates a fake CVS repository with multiple branches.
102 | #
103 | # \param repository Path to the repository to create.
104 | # \param ... Names of the branches to create.  Each branch will contain a file
105 | #     named "file" with different contents for each branch.
106 | create_git_repo_and_branches() {
107 |     local repository="${1}"
108 | 
109 |     # Git can, in theory, automatically set some default values for these.
110 |     # Unfortunately, such autoconfiguration can fail if e.g. the domain name
111 |     # for the machine is missing.  Be explicit.
112 |     git config --global user.email test@example.com
113 |     git config --global user.name Test
114 | 
115 |     git init "${repository}"
116 |     cd "${repository}"
117 | 
118 |     for branch in "${@}"; do
119 |         git checkout -b "${branch}"
120 |         echo "${branch} branch" >file
121 |         git add file
122 |         git commit -m "a revision"
123 |     done
124 | 
125 |     git checkout -b other
126 |     cd -
127 | }
128 | 
129 | 
130 | atf_test_case config__builtins
131 | config__builtins_body() {
132 |     mkdir sandbox-modules
133 |     export SANDBOXCTL_MODULESDIR="$(pwd)/sandbox-modules"
134 | 
135 |     cat >expout <<EOF
136 | AUTO_PACKAGES is undefined
137 | CVS_ROOT = :ext:anoncvs@anoncvs.NetBSD.org:/cvsroot
138 | CVS_TAG is undefined
139 | DISTDIR = /usr/pkgsrc/distfiles
140 | EXTRA_MKCONF is undefined
141 | FETCH_VCS = cvs
142 | GIT_BRANCH = trunk
143 | GIT_URL = https://github.com/NetBSD/pkgsrc.git
144 | LOCALBASE = /usr/pkg
145 | NJOBS = 99
146 | PACKAGES = /usr/pkgsrc/packages
147 | PBULK_LOG = /usr/pkgsrc/packages/log
148 | PBULK_PACKAGES = /usr/pkgsrc/packages/pbulk
149 | PKG_DBDIR = /usr/pkg/libdata/pkgdb
150 | PKG_DEVELOPER = no
151 | PKGSRCDIR = /usr/pkgsrc
152 | SANDBOX_CONFFILE is undefined
153 | SYSCONFDIR = /etc
154 | UPDATE_SOURCES = true
155 | VARBASE = /var
156 | 
157 | SANDBOX_ROOT is undefined
158 | SANDBOX_TYPE = empty
159 | EOF
160 |     atf_check -o file:expout env SHTK_HW_NCPUS=99 pkg_comp -c /dev/null config
161 | }
162 | 
163 | 
164 | atf_test_case config__path__components
165 | config__path__components_body() {
166 |     mkdir system
167 |     export PKG_COMP_ETCDIR="$(pwd)/system"
168 | 
169 |     echo "CVS_TAG=tag1" >my-file
170 |     atf_check -o match:"CVS_TAG = tag1" pkg_comp -c ./my-file config
171 | }
172 | 
173 | 
174 | atf_test_case config__path__extension
175 | config__path__extension_body() {
176 |     mkdir system
177 |     export PKG_COMP_ETCDIR="$(pwd)/system"
178 | 
179 |     echo "CVS_TAG=tag2" >my-file.conf
180 |     atf_check -o match:"CVS_TAG = tag2" pkg_comp -c my-file.conf config
181 | }
182 | 
183 | 
184 | atf_test_case config__name__system_directory
185 | config__name__system_directory_body() {
186 |     mkdir system
187 |     export PKG_COMP_ETCDIR="$(pwd)/system"
188 | 
189 |     echo "CVS_ROOT='custom-root'" >system/foo.conf
190 |     atf_check -o match:"CVS_ROOT = custom-root" pkg_comp -c foo config
191 | }
192 | 
193 | 
194 | atf_test_case config__name__not_found
195 | config__name__not_found_body() {
196 |     mkdir system
197 |     export PKG_COMP_ETCDIR="$(pwd)/system"
198 | 
199 |     cat >experr <<EOF
200 | pkg_comp: E: Cannot locate configuration named 'foobar'
201 | Type 'man pkg_comp' for help
202 | EOF
203 |     atf_check -s exit:1 -o empty -e file:experr pkg_comp -c foobar config
204 | }
205 | 
206 | 
207 | atf_test_case config__overrides
208 | config__overrides_body() {
209 |     mkdir sandbox-modules
210 |     export SANDBOXCTL_MODULESDIR="$(pwd)/sandbox-modules"
211 | 
212 |     cat >custom.conf <<EOF
213 | CVS_ROOT=the-root
214 | CVS_TAG=the-tag
215 | PKGSRCDIR=/usr/pkgsrc
216 | EOF
217 |     cat >sandbox.conf <<EOF
218 | SANDBOX_ROOT=/non-existent/location
219 | EOF
220 | 
221 |     cat >expout <<EOF
222 | AUTO_PACKAGES is undefined
223 | CVS_ROOT = foo bar
224 | CVS_TAG = tag123
225 | DISTDIR = /usr/pkgsrc/distfiles
226 | EXTRA_MKCONF is undefined
227 | FETCH_VCS = cvs
228 | GIT_BRANCH = trunk
229 | GIT_URL = https://github.com/NetBSD/pkgsrc.git
230 | LOCALBASE = /usr/pkg
231 | NJOBS = 80
232 | PACKAGES = /usr/pkgsrc/packages
233 | PBULK_LOG = /usr/pkgsrc/packages/log
234 | PBULK_PACKAGES = /usr/pkgsrc/packages/pbulk
235 | PKG_DBDIR = /usr/pkg/libdata/pkgdb
236 | PKG_DEVELOPER = no
237 | PKGSRCDIR is undefined
238 | SANDBOX_CONFFILE = $(pwd)/sandbox.conf
239 | SYSCONFDIR = /etc
240 | UPDATE_SOURCES = true
241 | VARBASE = /var
242 | 
243 | SANDBOX_ROOT = /non-existent/location
244 | SANDBOX_TYPE = empty
245 | EOF
246 |     atf_check -o file:expout pkg_comp -c custom.conf -o CVS_ROOT="foo bar" \
247 |         -o CVS_TAG=tag123 -o NJOBS=80 -o PKGSRCDIR= \
248 |         -o SANDBOX_CONFFILE="$(pwd)/sandbox.conf" config
249 | }
250 | 
251 | 
252 | atf_test_case config__too_many_args
253 | config__too_many_args_body() {
254 |     cat >experr <<EOF
255 | pkg_comp: E: config does not take any arguments
256 | Type 'man pkg_comp' for help
257 | EOF
258 |     atf_check -s exit:1 -e file:experr pkg_comp -c /dev/null config foo
259 | }
260 | 
261 | 
262 | atf_test_case fetch__cvs__checkout
263 | fetch__cvs__checkout_head() {
264 |     atf_set require.progs cvs
265 | }
266 | fetch__cvs__checkout_body() {
267 |     create_mock_cvsroot "${MOCK_CVSROOT}"
268 |     cat >test.conf <<EOF
269 | CVS_ROOT="${MOCK_CVSROOT}"
270 | PKGSRCDIR="$(pwd)/checkout/pkgsrc"
271 | EOF
272 | 
273 |     atf_check -o ignore -e ignore pkg_comp -c test.conf fetch
274 |     test -f checkout/pkgsrc/file-in-pkgsrc || atf_fail "pkgsrc not checked out"
275 | }
276 | 
277 | 
278 | atf_test_case fetch__cvs__update
279 | fetch__cvs__update_head() {
280 |     atf_set require.progs cvs
281 | }
282 | fetch__cvs__update_body() {
283 |     create_mock_cvsroot "${MOCK_CVSROOT}"
284 |     cat >test.conf <<EOF
285 | CVS_ROOT="${MOCK_CVSROOT}"
286 | PKGSRCDIR="$(pwd)/checkout/pkgsrc"
287 | EOF
288 | 
289 |     mkdir checkout
290 |     cd checkout
291 |     atf_check -o ignore -e ignore cvs -d"${MOCK_CVSROOT}" checkout -P pkgsrc
292 |     cd -
293 | 
294 |     cp -rf checkout/pkgsrc pkgsrc-copy
295 |     cd pkgsrc-copy
296 |     echo "second revision" >file-in-pkgsrc
297 |     cvs commit -m "Second revision."
298 |     cd -
299 | 
300 |     test -f checkout/pkgsrc/file-in-pkgsrc || atf_fail "pkgsrc not present yet"
301 |     if grep "second revision" checkout/pkgsrc/file-in-pkgsrc >/dev/null; then
302 |         atf_fail "second revision already present"
303 |     fi
304 | 
305 |     atf_check -o ignore -e ignore pkg_comp -c test.conf fetch
306 | 
307 |     grep "second revision" checkout/pkgsrc/file-in-pkgsrc >/dev/null \
308 |         || atf_fail "pkgsrc not updated"
309 | }
310 | 
311 | 
312 | atf_test_case fetch__git__clone
313 | fetch__git__clone_head() {
314 |     atf_set require.progs git
315 | }
316 | fetch__git__clone_body() {
317 |     create_git_repo_and_branches repo trunk1 trunk2
318 | 
319 |     cat >test.conf <<EOF
320 | FETCH_VCS=git
321 | GIT_BRANCH=trunk2
322 | GIT_URL="file://$(pwd)/repo"
323 | PKGSRCDIR="$(pwd)/checkout/pkgsrc"
324 | EOF
325 | 
326 |     atf_check -o ignore -e ignore pkg_comp -c test.conf fetch
327 |     atf_check -o inline:'trunk2 branch\n' cat checkout/pkgsrc/file
328 | }
329 | 
330 | 
331 | atf_test_case fetch__git__update
332 | fetch__git__update_head() {
333 |     atf_set require.progs git
334 | }
335 | fetch__git__update_body() {
336 |     create_git_repo_and_branches repo trunk1 trunk2
337 | 
338 |     mkdir checkout
339 |     atf_check -o ignore -e ignore \
340 |         git clone -b trunk2 "file://$(pwd)/repo" checkout/pkgsrc
341 | 
342 |     cat >test.conf <<EOF
343 | FETCH_VCS=git
344 | GIT_BRANCH=trunk1
345 | GIT_URL="file://$(pwd)/repo"
346 | PKGSRCDIR="$(pwd)/checkout/pkgsrc"
347 | EOF
348 | 
349 |     atf_check -o inline:'trunk2 branch\n' cat checkout/pkgsrc/file
350 |     atf_check -o ignore -e ignore pkg_comp -c test.conf fetch
351 |     atf_check -o inline:'trunk1 branch\n' cat checkout/pkgsrc/file
352 | }
353 | 
354 | 
355 | atf_test_case fetch__hooks__ok
356 | fetch__hooks__ok_head() {
357 |     atf_set require.progs cvs
358 | }
359 | fetch__hooks__ok_body() {
360 |     create_mock_cvsroot "${MOCK_CVSROOT}"
361 |     cat >test.conf <<EOF
362 | CVS_ROOT="${MOCK_CVSROOT}"
363 | PKGSRCDIR="$(pwd)/checkout/pkgsrc"
364 | 
365 | post_fetch_hook() {
366 |     test -d "\${PKGSRCDIR}" || return 1
367 |     echo "Hook after fetch"
368 | }
369 | EOF
370 | 
371 |     atf_check -o save:stdout -e ignore pkg_comp -c test.conf fetch
372 |     test -f checkout/pkgsrc/file-in-pkgsrc || atf_fail "pkgsrc not checked out"
373 | 
374 |     cat >exp_order <<EOF
375 | Hook after fetch
376 | EOF
377 |     atf_check -o file:exp_order grep '^Hook' stdout
378 | }
379 | 
380 | 
381 | atf_test_case fetch__hooks__post_fail
382 | fetch__hooks__post_fail_head() {
383 |     atf_set require.progs cvs
384 | }
385 | fetch__hooks__post_fail_body() {
386 |     create_mock_cvsroot "${MOCK_CVSROOT}"
387 |     cat >test.conf <<EOF
388 | CVS_ROOT="${MOCK_CVSROOT}"
389 | PKGSRCDIR="$(pwd)/checkout/pkgsrc"
390 | 
391 | post_fetch_hook() {
392 |     echo "Hook after fetch"
393 |     false
394 | }
395 | EOF
396 | 
397 |     atf_check -s exit:1 -o save:stdout -e save:stderr \
398 |         pkg_comp -c test.conf fetch
399 |     test -f checkout/pkgsrc/file-in-pkgsrc || atf_fail "pkgsrc not checked out"
400 |     grep 'post_fetch_hook returned an error' stderr || \
401 |         atf_fail "post_fetch_hook didn't seem to fail"
402 | 
403 |     cat >exp_order <<EOF
404 | Hook after fetch
405 | EOF
406 |     atf_check -o file:exp_order grep '^Hook' stdout
407 | }
408 | 
409 | 
410 | atf_test_case fetch__too_many_args
411 | fetch__too_many_args_body() {
412 |     cat >experr <<EOF
413 | pkg_comp: E: fetch does not take any arguments
414 | Type 'man pkg_comp' for help
415 | EOF
416 |     atf_check -s exit:1 -e file:experr pkg_comp -c /dev/null fetch foo
417 | }
418 | 
419 | 
420 | atf_test_case sandboxctl__all_commands_delegation
421 | sandboxctl__all_commands_delegation_body() {
422 |     cat >sandboxctl <<EOF
423 | #! /bin/sh
424 | printf args:
425 | while [ \${#} -gt 0 ]; do
426 |     printf " ~\${1}~"
427 |     shift
428 | done
429 | echo
430 | exit 42
431 | EOF
432 |     chmod +x sandboxctl
433 |     export SANDBOXCTL="$(pwd)/sandboxctl"
434 | 
435 |     for cmd in config create destroy mount run shell unmount; do
436 |         atf_check -s exit:42 -o save:out pkg_comp -c /dev/null \
437 |             "sandbox-${cmd}" abc
438 |         echo "args: ~-c${TMPDIR}/pkg_comp.ZZZZZZ~ ~${cmd}~ ~abc~" >expout
439 |         atf_check -o file:expout \
440 |             sed -e "s,/pkg_comp\.......,/pkg_comp.ZZZZZZ," out
441 |     done
442 | 
443 |     atf_check -s exit:42 -o save:out pkg_comp -c /dev/null -v sandbox-create
444 |     echo "args: ~-c${TMPDIR}/pkg_comp.ZZZZZZ~ ~-v~ ~create~" >expout
445 |     atf_check -o file:expout sed -e "s,/pkg_comp\.......,/pkg_comp.ZZZZZZ," out
446 | }
447 | 
448 | 
449 | atf_test_case sandboxctl__create_destroy_integration
450 | sandboxctl__create_destroy_integration_body() {
451 |     cat >pkg_comp.conf <<EOF
452 | DISTDIR="$(pwd)/distfiles"
453 | PACKAGES="$(pwd)/packages/pkg"
454 | PBULK_LOG="$(pwd)/packages/log"
455 | PBULK_PACKAGES="$(pwd)/packages/pbulk"
456 | PKGSRCDIR="$(pwd)/pkgsrc"
457 | SANDBOX_CONFFILE="$(pwd)/sandbox.conf"
458 | EOF
459 |     cat >sandbox.conf <<EOF
460 | SANDBOX_ROOT="$(pwd)/sandbox"
461 | SANDBOX_TYPE=empty
462 | EOF
463 | 
464 |     atf_check -s exit:1 -e match:'sandboxctl.* create.* arguments' \
465 |         pkg_comp -c pkg_comp.conf sandbox-create foo bar
466 | 
467 |     for dir in distfiles packages pkgsrc sandbox; do
468 |         [ ! -d "${dir}" ] || atf_fail "${dir} should not yet exist"
469 |     done
470 | 
471 |     atf_check pkg_comp -c pkg_comp.conf sandbox-create
472 |     for dir in distfiles packages pkgsrc sandbox; do
473 |         [ -d "${dir}" ] || atf_fail "${dir} was not created"
474 |     done
475 |     atf_check pkg_comp -c pkg_comp.conf sandbox-destroy
476 |     [ -d distfiles ] || atf_fail "distfiles was destroyed by mistake"
477 |     [ -d packages ] || atf_fail "packages was destroyed by mistake"
478 |     [ -d pkgsrc ] || atf_fail "pkgsrc was destroyed by mistake"
479 |     [ ! -d sandbox ] || atf_fail "sandbox was not destroyed"
480 | }
481 | 
482 | 
483 | atf_test_case no_command
484 | no_command_body() {
485 |     cat >experr <<EOF
486 | pkg_comp: E: No command specified
487 | Type 'man pkg_comp' for help
488 | EOF
489 |     atf_check -s exit:1 -e file:experr pkg_comp
490 | }
491 | 
492 | 
493 | atf_test_case unknown_command
494 | unknown_command_body() {
495 |     cat >experr <<EOF
496 | pkg_comp: E: Unknown command foo
497 | Type 'man pkg_comp' for help
498 | EOF
499 |     atf_check -s exit:1 -e file:experr pkg_comp foo
500 | }
501 | 
502 | 
503 | atf_test_case unknown_flag
504 | unknown_flag_body() {
505 |     cat >experr <<EOF
506 | pkg_comp: E: Unknown option -Z
507 | Type 'man pkg_comp' for help
508 | EOF
509 |     atf_check -s exit:1 -e file:experr pkg_comp -Z
510 | }
511 | 
512 | 
513 | atf_init_test_cases() {
514 |     atf_add_test_case config__builtins
515 |     atf_add_test_case config__path__components
516 |     atf_add_test_case config__path__extension
517 |     atf_add_test_case config__name__system_directory
518 |     atf_add_test_case config__name__not_found
519 |     atf_add_test_case config__overrides
520 |     atf_add_test_case config__too_many_args
521 | 
522 |     atf_add_test_case fetch__cvs__checkout
523 |     atf_add_test_case fetch__cvs__update
524 |     atf_add_test_case fetch__git__clone
525 |     atf_add_test_case fetch__git__update
526 |     atf_add_test_case fetch__hooks__ok
527 |     atf_add_test_case fetch__hooks__post_fail
528 |     atf_add_test_case fetch__too_many_args
529 | 
530 |     atf_add_test_case sandboxctl__all_commands_delegation
531 |     atf_add_test_case sandboxctl__create_destroy_integration
532 | 
533 |     atf_add_test_case no_command
534 |     atf_add_test_case unknown_command
535 |     atf_add_test_case unknown_flag
536 | }
537 | 


--------------------------------------------------------------------------------
/pkg_comp.sh:
--------------------------------------------------------------------------------
  1 | # Copyright 2013 Google Inc.
  2 | # All rights reserved.
  3 | #
  4 | # Redistribution and use in source and binary forms, with or without
  5 | # modification, are permitted provided that the following conditions are
  6 | # met:
  7 | #
  8 | # * Redistributions of source code must retain the above copyright
  9 | #   notice, this list of conditions and the following disclaimer.
 10 | # * Redistributions in binary form must reproduce the above copyright
 11 | #   notice, this list of conditions and the following disclaimer in the
 12 | #   documentation and/or other materials provided with the distribution.
 13 | # * Neither the name of Google Inc. nor the names of its contributors
 14 | #   may be used to endorse or promote products derived from this software
 15 | #   without specific prior written permission.
 16 | #
 17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 18 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 19 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 20 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 21 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 22 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 23 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 24 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 25 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 26 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 28 | 
 29 | # \file pkg_comp.sh
 30 | # Entry point and main program logic.
 31 | 
 32 | shtk_import cleanup
 33 | shtk_import cli
 34 | shtk_import config
 35 | shtk_import cvs
 36 | shtk_import hw
 37 | : ${PKG_COMP_SHTK_MODULESDIR:="__PKG_COMP_SHTK_MODULESDIR__"}
 38 | SHTK_MODULESPATH="${PKG_COMP_SHTK_MODULESDIR}" shtk_import git
 39 | SHTK_MODULESPATH="${PKG_COMP_SHTK_MODULESDIR}" shtk_import pkgsrc
 40 | 
 41 | 
 42 | # List of valid configuration variables.
 43 | #
 44 | # Please remember to update pkg_comp.conf(5) if you change this list.
 45 | PKG_COMP_CONFIG_VARS="AUTO_PACKAGES CVS_ROOT CVS_TAG DISTDIR EXTRA_MKCONF
 46 |                       FETCH_VCS GIT_BRANCH GIT_URL LOCALBASE NJOBS PACKAGES
 47 |                       PBULK_LOG PBULK_PACKAGES PKG_DBDIR PKG_DEVELOPER PKGSRCDIR
 48 |                       SANDBOX_CONFFILE SYSCONFDIR UPDATE_SOURCES VARBASE"
 49 | 
 50 | 
 51 | # Paths to installed files.
 52 | #
 53 | # Can be overriden for test purposes only.
 54 | : ${PKG_COMP_ETCDIR="__PKG_COMP_ETCDIR__"}
 55 | : ${PKG_COMP_SHAREDIR="__PKG_COMP_SHAREDIR__"}
 56 | : ${SANDBOXCTL="__SANDBOXCTL__"}
 57 | 
 58 | 
 59 | # Sets defaults for configuration variables and hooks that need to exist.
 60 | #
 61 | # This function should be before the configuration file has been loaded.  This
 62 | # means that the user can undefine a required configuration variable, but we let
 63 | # him shoot himself in the foot if he so desires.
 64 | pkg_comp_set_defaults() {
 65 |     # Please remember to update pkg_comp.conf(5) if you change any default
 66 |     # values.
 67 |     shtk_config_set CVS_ROOT ":ext:anoncvs@anoncvs.NetBSD.org:/cvsroot"
 68 |     shtk_config_set DISTDIR "/usr/pkgsrc/distfiles"
 69 |     shtk_config_set FETCH_VCS "cvs"
 70 |     shtk_config_set GIT_BRANCH "trunk"
 71 |     shtk_config_set GIT_URL "https://github.com/NetBSD/pkgsrc.git"
 72 |     shtk_config_set LOCALBASE "/usr/pkg"
 73 |     shtk_config_set NJOBS "$(shtk_hw_ncpus)"
 74 |     shtk_config_set PACKAGES "/usr/pkgsrc/packages"
 75 |     shtk_config_set PBULK_LOG "/usr/pkgsrc/packages/log"
 76 |     shtk_config_set PBULK_PACKAGES "/usr/pkgsrc/packages/pbulk"
 77 |     shtk_config_set PKG_DBDIR "/usr/pkg/libdata/pkgdb"
 78 |     shtk_config_set PKG_DEVELOPER "no"
 79 |     shtk_config_set PKGSRCDIR "/usr/pkgsrc"
 80 |     shtk_config_set SYSCONFDIR "/etc"
 81 |     shtk_config_set UPDATE_SOURCES "true"
 82 |     shtk_config_set VARBASE "/var"
 83 | 
 84 |     post_fetch_hook() { true; }
 85 | }
 86 | 
 87 | 
 88 | # Executes sandboxctl based on the pkg_comp configuration.
 89 | #
 90 | # \params ... The options and arguments to forward to sandboxctl.
 91 | #
 92 | # \return The exit code of sandboxctl.
 93 | run_sandboxctl() {
 94 |     # Generate a temporary configuration file for sandboxctl that stitches
 95 |     # together our default hooks and the user-provided configuration.
 96 |     #
 97 |     # Note: it feels tempting to mix together the contents of pkg_comp.conf
 98 |     # and sandboxctl.conf into the same file because their variables are
 99 |     # disjoint.  However, doing so is extremely confusing due to the way
100 |     # shtk_config works: for example, not all global variables would be visible
101 |     # in all hooks, and the -o flag would not be able to override sandbox.conf
102 |     # settings.  Therefore, while it feels uglier to keep things separate, it
103 |     # will avoid confusion down the road.
104 |     if [ -z "${_PKG_COMP_SANDBOXCTL_CONFIG_FILE}" ]; then
105 |         local userconf=
106 |         if shtk_config_has SANDBOX_CONFFILE; then
107 |             userconf="$(shtk_config_get SANDBOX_CONFFILE)"
108 |             [ -e "${userconf}" ] || shtk_cli_error "sandbox configuration" \
109 |                  "file ${userconf} does not exist"
110 |         fi
111 | 
112 |         local pattern="${TMPDIR:-/tmp}/pkg_comp.XXXXXX"
113 |         _PKG_COMP_SANDBOXCTL_CONFIG_FILE="$(mktemp "${pattern}" 2>/dev/null)"
114 |         remove_conffile() {
115 |             rm -f "${_PKG_COMP_SANDBOXCTL_CONFIG_FILE}"
116 |         }
117 |         shtk_cleanup_register remove_conffile
118 | 
119 |         {
120 |             cat "${PKG_COMP_SHAREDIR}/sandbox.conf.pre"
121 |             cat <<EOF
122 | DISTDIR="$(shtk_config_has DISTDIR && shtk_config_get DISTDIR)"
123 | PACKAGES="$(shtk_config_has PACKAGES && shtk_config_get PACKAGES)"
124 | PBULK_LOG="$(shtk_config_has PBULK_LOG && shtk_config_get PBULK_LOG)"
125 | PBULK_PACKAGES="$(shtk_config_has PBULK_PACKAGES \
126 |     && shtk_config_get PBULK_PACKAGES)"
127 | PKGSRCDIR="$(shtk_config_has PKGSRCDIR && shtk_config_get PKGSRCDIR)"
128 | PKG_DEVELOPER="$(shtk_config_has PKG_DEVELOPER \
129 |     && shtk_config_get PKG_DEVELOPER)"
130 | EOF
131 |             [ -z "${userconf}" ] || cat "${userconf}"
132 |             cat "${PKG_COMP_SHAREDIR}/sandbox.conf.post"
133 |         } >>"${_PKG_COMP_SANDBOXCTL_CONFIG_FILE}" \
134 |             || shtk_cli_error "Failed to create sandbox.conf file"
135 |     fi
136 | 
137 |     local vflag=
138 |     if shtk_cli_log_level debug; then
139 |         vflag=-v
140 |     fi
141 | 
142 |     "${SANDBOXCTL}" -c"${_PKG_COMP_SANDBOXCTL_CONFIG_FILE}" ${vflag} "${@}"
143 | }
144 | 
145 | 
146 | # Configures the bootstrap kit, either from a prebuilt binary or from sources.
147 | #
148 | # Does nothing if the bootstrap kit has already been configured.
149 | #
150 | # \param basename Unique name for the pkg tree being configured.
151 | # \param root Path to the root of the sandbox.
152 | # \param packages Value of PACKAGES for this bootstrap kit.
153 | # \param pkgdbdir Value of PKG_DBDIR for this bootstrap kit.
154 | # \param prefix Value of PREFIX for this bootstrap kit.
155 | # \param sysconfdir Value of SYSCONFDIR for this bootstrap kit.
156 | # \param varbase Value of VARBASE for this bootstrap kit.
157 | setup_bootstrap() {
158 |     local basename="${1}"; shift
159 |     local root="${1}"; shift
160 |     local packages="${1}"; shift
161 |     local pkgdbdir="${1}"; shift
162 |     local prefix="${1}"; shift
163 |     local sysconfdir="${1}"; shift
164 |     local varbase="${1}"; shift
165 | 
166 |     [ ! -f "${root}${prefix}/sbin/pkg_admin" ] || return 0
167 | 
168 |     local binarykit="${packages}/bootstrap.tgz"
169 |     if [ -e "${binarykit}" ]; then
170 |         shtk_cli_info "Setting up bootstrap in ${prefix} using binary kit"
171 |         run_sandboxctl run /bin/sh -c \
172 |             "cd / && tar xzpf /pkg_comp/packages/${basename}/bootstrap.tgz" \
173 |             || exit
174 |     else
175 |         # Wipe any previous bootstrap work directory.  This is helpful in
176 |         # case a bootstrap execution failed for reasons unknown to us and
177 |         # the user wants to retry without recreating the sandbox from scratch.
178 |         rm -rf "${root}/pkg_comp/work/${basename}/bootstrap"
179 | 
180 |         local bootstrap_sh=/bin/sh
181 |         local dash_echo_test="$("${bootstrap_sh}" -c 'echo "\\100"')"
182 |         if [ "$(uname -s)" != GNUkFreeBSD -a "${dash_echo_test}" = @ ]; then
183 |             local bash
184 |             bash="$(which bash)"
185 |             if [ ${?} -eq 0 ]; then
186 |                 bootstrap_sh="${bash}"
187 |             else
188 |                 shtk_cli_warning "/bin/sh appears to be dash and bash was not" \
189 |                     "found; bootstrap will probably fail"
190 |             fi
191 |         fi
192 | 
193 |         shtk_cli_info "Setting up bootstrap in ${prefix} from scratch"
194 |         run_sandboxctl run /bin/sh -c \
195 |             "cd /pkg_comp/pkgsrc/bootstrap && env \
196 |                  DISTDIR=/pkg_comp/distfiles \
197 |                  PACKAGES='/pkg_comp/packages/${basename}' \
198 |                  SH='${bootstrap_sh}' \
199 |                  ./bootstrap \
200 |                  --gzip-binary-kit=/pkg_comp/packages/${basename}/bootstrap.tgz\
201 |                  --make-jobs='$(shtk_config_get NJOBS)' \
202 |                  --pkgdbdir='${pkgdbdir}' \
203 |                  --prefix='${prefix}' \
204 |                  --sysconfdir='${sysconfdir}' \
205 |                  --varbase='${varbase}' \
206 |                  --workdir=/pkg_comp/work/${basename}/bootstrap" \
207 |             || exit
208 |     fi
209 | }
210 | 
211 | 
212 | # Configures bmake within the sandbox.
213 | #
214 | # Creates a mk.conf file with pkg_comp-specific details, and also creates a
215 | # convenience symlink in a fixed location for interactive uses of make.
216 | #
217 | # Does nothing if bmake has already been configured.
218 | #
219 | # \param basename Unique name for the pkg tree being configured.
220 | # \param root Path to the root of the sandbox.
221 | # \param sysconfdir Value of SYSCONFDIR for this bootstrap kit.
222 | setup_make() {
223 |     local basename="${1}"; shift
224 |     local root="${1}"; shift
225 |     local sysconfdir="${1}"; shift
226 | 
227 |     local mk_conf="/pkg_comp/${basename}.mk.conf"
228 |     local include=".include \"${mk_conf}\""
229 |     if ! grep -qF "${include}" "${root}${sysconfdir}/mk.conf"; then
230 |         echo ".ifdef BSD_PKG_MK" >>"${root}${sysconfdir}/mk.conf"
231 |         echo "${include}" >>"${root}${sysconfdir}/mk.conf"
232 |         echo ".endif" >>"${root}${sysconfdir}/mk.conf"
233 |     fi
234 | 
235 |     local symlink="/pkg_comp/make-${basename}"
236 |     [ ! -e "${root}${mk_conf}" -o ! -e "${root}${symlink}" ] || return 0
237 | 
238 |     shtk_cli_info "Setting up ${mk_conf}"
239 |     cat >"${root}${mk_conf}" <<EOF
240 | DISTDIR=/pkg_comp/distfiles
241 | PACKAGES=/pkg_comp/packages/${basename}
242 | PKGSRCDIR=/pkg_comp/pkgsrc
243 | WRKOBJDIR=/pkg_comp/work/${basename}
244 | 
245 | BINPKG_SITES=
246 | DEPENDS_TARGET=bin-install
247 | 
248 | MAKE_JOBS=$(shtk_config_get NJOBS)
249 | 
250 | PKG_DEVELOPER=$(shtk_config_get PKG_DEVELOPER)
251 | EOF
252 | 
253 |     ln -s "${prefix}/bin/bmake" "${root}${symlink}"
254 | }
255 | 
256 | 
257 | # Configures pkg_install within the sandbox.
258 | #
259 | # Creates a pkg_install.conf file with pkg_comp-specific details.
260 | #
261 | # Does nothing if pkg_install has already been configured.
262 | #
263 | # \param basename Unique name for the pkg tree being configured.
264 | # \param root Path to the root of the sandbox.
265 | # \param sysconfdir Value of SYSCONFDIR for this bootstrap kit.
266 | # \param pkgdbdir Value of PKG_DBDIR for this bootstrap kit.
267 | setup_pkginstall() {
268 |     local basename="${1}"; shift
269 |     local root="${1}"; shift
270 |     local sysconfdir="${1}"; shift
271 |     local pkgdbdir="${1}"; shift
272 | 
273 |     local pkginstall_conf="${sysconfdir}/pkg_install.conf"
274 |     [ ! -e "${root}${pkginstall_conf}" ] || return 0
275 | 
276 |     shtk_cli_info "Setting up ${pkginstall_conf}"
277 |     cat >"${root}${pkginstall_conf}" <<EOF
278 | PKG_DBDIR=${pkgdbdir}
279 | PKG_PATH=/pkg_comp/packages/${basename}/All
280 | EOF
281 | }
282 | 
283 | 
284 | # Sets up a pkg installation, including bootstrap, bmake, and pkg_install.
285 | #
286 | # This is idempotent: once the installation has been configured, this function
287 | # does nothing.  This is good for speed, but can lead to inconsistencies if the
288 | # user changes the configuration without rebuilding the sandbox.
289 | #
290 | # \param basename Unique name for the pkg tree being configured.
291 | # \param root Path to the root of the sandbox.
292 | # \param packages Value of PACKAGES for this bootstrap kit.
293 | # \param pkgdbdir Value of PKG_DBDIR for this bootstrap kit.
294 | # \param prefix Value of PREFIX for this bootstrap kit.
295 | # \param sysconfdir Value of SYSCONFDIR for this bootstrap kit.
296 | # \param varbase Value of VARBASE for this bootstrap kit.
297 | full_bootstrap() {
298 |     local basename="${1}"; shift
299 |     local root="${1}"; shift
300 |     local packages="${1}"; shift
301 |     local pkgdbdir="${1}"; shift
302 |     local prefix="${1}"; shift
303 |     local sysconfdir="${1}"; shift
304 |     local varbase="${1}"; shift
305 | 
306 |     setup_bootstrap "${basename}" "${root}" "${packages}" "${pkgdbdir}" \
307 |         "${prefix}" "${sysconfdir}" "${varbase}"
308 |     setup_make "${basename}" "${root}" "${sysconfdir}"
309 |     setup_pkginstall "${basename}" "${root}" "${sysconfdir}" "${pkgdbdir}"
310 | }
311 | 
312 | 
313 | # Sets up a pkg installation in the location configured by the user.
314 | #
315 | # \param root Path to the root of the sandbox.
316 | bootstrap_pkg() {
317 |     local root="${1}"; shift
318 | 
319 |     full_bootstrap \
320 |         "pkg" \
321 |         "${root}" \
322 |         "$(shtk_config_get PACKAGES)" \
323 |         "$(shtk_config_get PKG_DBDIR)" \
324 |         "$(shtk_config_get LOCALBASE)" \
325 |         "$(shtk_config_get SYSCONFDIR)" \
326 |         "$(shtk_config_get VARBASE)" || exit
327 | 
328 |     if shtk_config_has EXTRA_MKCONF; then
329 |         local extra_mkconf="$(shtk_config_get EXTRA_MKCONF)"
330 |         cat "$(shtk_config_get EXTRA_MKCONF)" >>"${root}/pkg_comp/pkg.mk.conf" \
331 |             || shtk_cli_error "Failed to append ${extra_mkconf} to mk.conf"
332 |     fi
333 | }
334 | 
335 | 
336 | # Sets up a pkg installation for pbulk and also configures pbulk.
337 | #
338 | # \param root Path to the root of the sandbox.
339 | bootstrap_pbulk() {
340 |     local root="${1}"; shift
341 | 
342 |     full_bootstrap \
343 |         "pbulk" \
344 |         "${root}" \
345 |         "$(shtk_config_get PBULK_PACKAGES)" \
346 |         "/pkg_comp/pbulk/libdata/pkgdb" \
347 |         "/pkg_comp/pbulk" \
348 |         "/pkg_comp/pbulk/etc" \
349 |         "/pkg_comp/pbulk/var" || exit
350 | 
351 |     cat >>"${root}/pkg_comp/pbulk.mk.conf" <<EOF
352 | # Be permissive of warnings raised during the build of our own infrastructure.
353 | # Linux is especially picky and it's easy to trip over different warnings on
354 | # different platforms.  We just don't want to abort the bootstrapping process
355 | # for such a lame reason.
356 | BUILDLINK_TRANSFORM+=rm:-Werror
357 | EOF
358 | 
359 |     run_sandboxctl run /bin/sh -c \
360 |         "cd /pkg_comp/pkgsrc/pkgtools/pbulk && \
361 |          /pkg_comp/pbulk/bin/bmake bin-install" || exit
362 |     # bin-install implies clean when building from source, so no need to do that
363 |     # on our own.
364 | 
365 |     # Make sure pbulk.conf exists in the etc directory in case automatic
366 |     # installation of configuration files is disabled.
367 |     local pbulk_conf="${root}/pkg_comp/pbulk/etc/pbulk.conf"
368 |     cp "${root}/pkg_comp/pbulk/share/examples/pbulk/pbulk.conf" "${pbulk_conf}"
369 | 
370 |     # Replaces the value of a single variable in pbulk.conf.
371 |     #
372 |     # This is inefficient and possibly fragile, but we cannot just append
373 |     # overrides to the end of the file because some variables are derived from
374 |     # others.
375 |     pbulk_set() {
376 |         local var="${1}"; shift
377 |         local value="${1}"; shift
378 | 
379 |         if grep "^${var}=" "${pbulk_conf}" >/dev/null; then
380 |             sed "/^${var}=/s,^.*$,${var}='${value}'," "${pbulk_conf}" \
381 |                 >"${pbulk_conf}.new"
382 |             mv "${pbulk_conf}.new" "${pbulk_conf}"
383 |         else
384 |             echo "${var}=${value}" >>"${pbulk_conf}"
385 |         fi
386 |     }
387 | 
388 |     pbulk_set checksum_packages no
389 |     pbulk_set mail true
390 |     pbulk_set master_mode no
391 |     pbulk_set rsync true
392 |     pbulk_set unprivileged_user root
393 | 
394 |     # Configure pbulk's file layout.
395 |     pbulk_set bulklog /pkg_comp/bulklog
396 |     pbulk_set limited_list /pkg_comp/pbulk/etc/pbulk.list
397 |     pbulk_set loc /pkg_comp/bulklog/SUMMARY
398 | 
399 |     # Configure pkgsrc's file layout.
400 |     pbulk_set bootstrapkit "/pkg_comp/packages/pkg/bootstrap.tgz"
401 |     pbulk_set packages "/pkg_comp/packages/pkg"
402 |     pbulk_set pkgdb "$(shtk_config_get PKG_DBDIR)"
403 |     pbulk_set pkgsrc /pkg_comp/pkgsrc
404 |     pbulk_set prefix "$(shtk_config_get LOCALBASE)"
405 |     pbulk_set varbase "$(shtk_config_get VARBASE)"
406 | 
407 |     # Extend pbulk's client-clean script so that, once the bootstrap kit has
408 |     # been extracted, we can modify the system-wide mk.conf to pull in our
409 |     # chroot-specific settings.
410 |     #
411 |     # TODO(jmmv): Ideally, we would either generate a wrapper script for
412 |     # client-clean and source it from the configuration (but that's not possible
413 |     # because some pbulk scripts hardcode the script they run), or we would
414 |     # extend pbulk to support some form of "mk-fragment" option like the
415 |     # bootstrap setup does.
416 |     cat >>"${root}/pkg_comp/pbulk/libexec/pbulk/client-clean" <<EOF
417 | cat >>"$(shtk_config_get SYSCONFDIR)/mk.conf" <<SUBEOF
418 | .ifdef BSD_PKG_MK
419 | .include "/pkg_comp/pkg.mk.conf"
420 | .endif
421 | SUBEOF
422 | EOF
423 | }
424 | 
425 | 
426 | # Generates pkg_summary for all built packages.
427 | #
428 | # pbulk does this on its own, but it only adds the packages that have just been
429 | # built to pkg_summary.  For pkg_comp, because we want to support the case where
430 | # users request individual additional packages to be built, we have to
431 | # regenerate pkg_summary using *all* existing binary packages and not only those
432 | # that were handled by pbulk.
433 | #
434 | # \param root Path to the root of the sandbox.
435 | generate_pkg_summary() {
436 |     local root="${1}"; shift
437 | 
438 |     shtk_cli_info "Generating pkg_summary"
439 |     # We need to run this within the sandbox because we have to execute pbulk's
440 |     # copy of pkg_info.
441 |     cat >"${root}/tmp/pkg_summary.sh" <<EOF
442 | #! /bin/sh
443 | cd /pkg_comp/packages/pkg/All
444 | rm -f pkg_summary.*
445 | ls -1 *.tgz | xargs /pkg_comp/pbulk/sbin/pkg_info -X | gzip -c >pkg_summary.gz
446 | gzip -dc <pkg_summary.gz | bzip2 -c >pkg_summary.bz2
447 | EOF
448 |     chmod +x "${root}/tmp/pkg_summary.sh"
449 |     run_sandboxctl run /tmp/pkg_summary.sh
450 | }
451 | 
452 | 
453 | # Automatic mode.
454 | #
455 | # Updates the pkgsrc tree, creates a sandbox, bootstraps pkgsrc, builds a set of
456 | # packages, and tears the sandbox down.
457 | #
458 | # \params ... The options and arguments to the command.
459 | pkg_comp_auto() {
460 |     local OPTIND  # Cope with bash failing to reinitialize getopt.
461 |     while getopts ':f' arg "${@}"; do
462 |         case "${arg}" in
463 |             f)  # Convenience flag for a "fast mode".
464 |                 shtk_config_set "UPDATE_SOURCES" "false"
465 |                 ;;
466 | 
467 |             \?)
468 |                 shtk_cli_usage_error "Unknown option -${OPTARG} in build"
469 |                 ;;
470 |         esac
471 |     done
472 |     shift $((${OPTIND} - 1))
473 | 
474 |     if [ ${#} -eq 0 ]; then
475 |         if shtk_config_has AUTO_PACKAGES; then
476 |             set -- $(shtk_config_get AUTO_PACKAGES)
477 |         fi
478 |     fi
479 |     [ ${#} -gt 0 ] || shtk_cli_usage_error "auto requires at least one" \
480 |         "package name as an argument or in the AUTO_PACKAGES variable"
481 | 
482 |     if shtk_config_get_bool UPDATE_SOURCES; then
483 |         pkg_comp_fetch
484 |     fi
485 | 
486 |     # We must validate packages after invoking pkg_comp_fetch to ensure the
487 |     # pkgsrc tree exists.
488 |     local packages
489 |     packages="$(pkgsrc_expand_packages "$(shtk_config_get PKGSRCDIR)" "${@}")" \
490 |         || shtk_cli_error "Some packages do not exist in pkgsrc; please fix" \
491 |             "and retry"
492 | 
493 |     local root
494 |     root="$(run_sandboxctl config SANDBOX_ROOT)" || exit
495 | 
496 |     if [ -d "${root}/pkg_comp" ]; then
497 |         shtk_cli_warning "Reusing existing sandbox; if this is not what you" \
498 |             "want to do, abort now with Ctrl-C and use sandbox-delete to" \
499 |             "wipe the previous sandbox"
500 |     else
501 |         cleanup() {
502 |             shtk_cli_info "Destroying sandbox"
503 |             run_sandboxctl destroy
504 |         }
505 |         shtk_cleanup_register cleanup
506 | 
507 |         shtk_cli_info "Creating sandbox"
508 |         run_sandboxctl create || exit
509 |     fi
510 | 
511 |     pkg_comp_build ${packages} || exit
512 | }
513 | 
514 | 
515 | # Bootstraps pkgsrc on the user's configured location and for pbulk.
516 | pkg_comp_bootstrap() {
517 |     [ ${#} -eq 0 ] \
518 |         || shtk_cli_usage_error "bootstrap does not take any arguments"
519 | 
520 |     local root
521 |     root="$(run_sandboxctl config SANDBOX_ROOT)" || exit
522 | 
523 |     [ ! -e "${root}/pkg_comp/done.bootstrap" ] || return 0
524 |     shtk_cli_info "Bootstrapping pkg tools"
525 |     bootstrap_pkg "${root}" || exit
526 |     shtk_cli_info "Bootstrapping pbulk tools"
527 |     bootstrap_pbulk "${root}" || exit
528 |     touch "${root}/pkg_comp/done.bootstrap"
529 | }
530 | 
531 | 
532 | # Builds one or more packages in an already-existing sandbox.
533 | #
534 | # \params ... The options and arguments to the command.
535 | pkg_comp_build() {
536 |     [ ${#} -gt 0 ] \
537 |         || shtk_cli_usage_error "build requires at least one package name"
538 | 
539 |     local packages
540 |     packages="$(pkgsrc_expand_packages "$(shtk_config_get PKGSRCDIR)" "${@}")" \
541 |         || shtk_cli_error "Some packages do not exist in pkgsrc; please fix" \
542 |             "and retry"
543 | 
544 |     pkg_comp_bootstrap || exit
545 | 
546 |     local root
547 |     root="$(run_sandboxctl config SANDBOX_ROOT)" || exit
548 | 
549 |     shtk_cli_info "Adding packages to be built to pbulk.list"
550 |     local list="${root}/pkg_comp/pbulk/etc/pbulk.list"
551 |     rm -f "${list}"
552 |     for package in ${packages}; do
553 |         echo "${package}" >>"${list}"
554 |     done
555 | 
556 |     local bulklog="$(shtk_config_get PBULK_LOG)"
557 | 
558 |     # Removing bulklog/success seems to be necessary to restart a build, as
559 |     # otherwise bulkbuild does not build modified packages.  Is this correct?
560 |     rm -f "${root}/pkg_comp/bulklog/success"
561 | 
562 |     shtk_cli_info "Starting pbulk build in the sandbox"
563 |     run_sandboxctl run /pkg_comp/pbulk/bin/bulkbuild || \
564 |         shtk_cli_error "bulkbuild failed; see ${bulklog}/ for possible details"
565 |     generate_pkg_summary "${root}"
566 | 
567 |     if [ -s "${bulklog}/SUMMARY/error" ]; then
568 |         local errors="$(cat ${bulklog}/SUMMARY/error)"
569 |         shtk_cli_error "Failed to build ${errors}; see ${bulklog}/ for" \
570 |             "detailed logs"
571 |     else
572 |         shtk_cli_info "All packages built successfully; summary reports are" \
573 |             "in ${bulklog}/SUMMARY/"
574 |     fi
575 | }
576 | 
577 | 
578 | # Dumps the loaded configuration.
579 | pkg_comp_config() {
580 |     [ ${#} -eq 0 ] || shtk_cli_usage_error "config does not take any arguments"
581 | 
582 |     for var in ${PKG_COMP_CONFIG_VARS}; do
583 |         if shtk_config_has "${var}"; then
584 |             echo "${var} = $(shtk_config_get "${var}")"
585 |         else
586 |             echo "${var} is undefined"
587 |         fi
588 |     done
589 | 
590 |     echo
591 |     run_sandboxctl config || exit
592 | }
593 | 
594 | 
595 | # Fetches a copy of the pkgsrc tree, or updates an existing one.
596 | pkg_comp_fetch() {
597 |     [ ${#} -eq 0 ] || shtk_cli_usage_error "fetch does not take any arguments"
598 | 
599 |     shtk_cli_info "Updating pkgsrc tree"
600 | 
601 |     case "$(shtk_config_get FETCH_VCS)" in
602 |         cvs)
603 |             shtk_cvs_fetch "$(shtk_config_get CVS_ROOT)" pkgsrc \
604 |                 "$(shtk_config_get_default CVS_TAG '')" \
605 |                 "$(shtk_config_get PKGSRCDIR)"
606 |             ;;
607 | 
608 |         git)
609 |             git_fetch "$(shtk_config_get GIT_URL)" \
610 |                 "$(shtk_config_get GIT_BRANCH)" \
611 |                 "$(shtk_config_get PKGSRCDIR)"
612 |             ;;
613 | 
614 |         *)
615 |             shtk_cli_error "Unknown VCS name in FETCH_VCS; found" \
616 |                 "$(shtk_config_get FETCH_VCS)"
617 |             ;;
618 |     esac
619 | 
620 |     shtk_config_run_hook post_fetch_hook
621 | }
622 | 
623 | 
624 | # Loads the configuration file specified in the command line.
625 | #
626 | # \param config_name Name of the desired configuration.  It can be either a
627 | #     configuration name (no slashes) or a path.
628 | pkg_comp_config_load() {
629 |     local config_name="${1}"; shift
630 | 
631 |     local config_file=
632 |     case "${config_name}" in
633 |         */*|*.conf)
634 |             config_file="${config_name}"
635 |             ;;
636 | 
637 |         *)
638 |             config_file="${PKG_COMP_ETCDIR}/${config_name}.conf"
639 |             [ -e "${config_file}" ] \
640 |                 || shtk_cli_usage_error "Cannot locate configuration named" \
641 |                 "'${config_name}'"
642 |             ;;
643 |     esac
644 |     shtk_config_load "${config_file}"
645 | }
646 | 
647 | 
648 | # Entry point to the program.
649 | #
650 | # \param ... Command-line arguments to be processed.
651 | #
652 | # \return An exit code to be returned to the user.
653 | main() {
654 |     local config_name="default"
655 | 
656 |     shtk_config_init ${PKG_COMP_CONFIG_VARS}
657 | 
658 |     local OPTIND
659 |     while getopts ':c:o:v' arg "${@}"; do
660 |         case "${arg}" in
661 |             c)  # Name of the configuration to load.
662 |                 config_name="${OPTARG}"
663 |                 ;;
664 | 
665 |             o)  # Override for a particular configuration variable.
666 |                 shtk_config_override "${OPTARG}"
667 |                 ;;
668 | 
669 |             v)  # Be verbose.
670 |                 shtk_cli_set_log_level debug
671 |                 ;;
672 | 
673 |             \?)
674 |                 shtk_cli_usage_error "Unknown option -${OPTARG}"
675 |                 ;;
676 |         esac
677 |     done
678 |     shift $((${OPTIND} - 1))
679 |     OPTIND=1  # Should not be necessary due to the 'local' above.
680 | 
681 |     [ ${#} -ge 1 ] || shtk_cli_usage_error "No command specified"
682 | 
683 |     local exit_code=0
684 | 
685 |     local command="${1}"; shift
686 |     case "${command}" in
687 |         auto|bootstrap|build|config|fetch)
688 |             pkg_comp_set_defaults
689 |             pkg_comp_config_load "${config_name}"
690 |             "pkg_comp_${command}" "${@}" || exit_code="${?}"
691 |             ;;
692 | 
693 |         sandbox-*)
694 |             pkg_comp_set_defaults
695 |             pkg_comp_config_load "${config_name}"
696 |             local subcommand="$(echo "${command}" | cut -d - -f 2-)"
697 |             run_sandboxctl "${subcommand}" "${@}" || exit_code="${?}"
698 |             ;;
699 | 
700 |         *)
701 |             shtk_cli_usage_error "Unknown command ${command}"
702 |             ;;
703 |     esac
704 | 
705 |     return "${exit_code}"
706 | }
707 | 


--------------------------------------------------------------------------------
/pkg_comp_inttest.sh:
--------------------------------------------------------------------------------
  1 | #! __ATF_SH__
  2 | # Copyright 2017 Google Inc.
  3 | # All rights reserved.
  4 | #
  5 | # Redistribution and use in source and binary forms, with or without
  6 | # modification, are permitted provided that the following conditions are
  7 | # met:
  8 | #
  9 | # * Redistributions of source code must retain the above copyright
 10 | #   notice, this list of conditions and the following disclaimer.
 11 | # * Redistributions in binary form must reproduce the above copyright
 12 | #   notice, this list of conditions and the following disclaimer in the
 13 | #   documentation and/or other materials provided with the distribution.
 14 | # * Neither the name of Google Inc. nor the names of its contributors
 15 | #   may be used to endorse or promote products derived from this software
 16 | #   without specific prior written permission.
 17 | #
 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 19 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 20 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 21 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 22 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 23 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 24 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 25 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 26 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 27 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 28 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 29 | 
 30 | # \file pkg_comp_inttest.sh
 31 | # Integration tests for the pkg_comp.sh script.
 32 | #
 33 | # The tests in this script are expensive to run: they use a real pkgsrc tree and
 34 | # a real sandbox, and test the various integration points of pkg_comp with the
 35 | # pkgsrc infrastructure (i.e. they build software).  These are "the real tests"
 36 | # because they ensure that pkgsrc works within the context of the sandbox we
 37 | # have created, and that pkg_comp is executing commands correctly; however, this
 38 | # also means that these can be fragile, as they are subject to pkgsrc API
 39 | # changes (good for us to spot problems) and breakage (bad as these are not our
 40 | # fault).
 41 | #
 42 | # The following configuration variables must be set:
 43 | #
 44 | #     test_suites.pkg_comp.distdir
 45 | #         Path to the persistent cache for distribution files.  This is safe to
 46 | #         reuse across test invocations.  Read-write.
 47 | #
 48 | #     test_suites.pkg_comp.packages
 49 | #         Path to a directory to hold built packages.  If not set, tests will
 50 | #         always start from a pristine state so they will be more robust.  If
 51 | #         set, tests may become flaky over time so set only if you know what you
 52 | #         are doing.  Read-write.
 53 | #
 54 | #     test_suites.pkg_comp.pkgsrcdir
 55 | #         Path to an pkgsrc tree cloned from Git.  Read-only.
 56 | #
 57 | #     test_suites.pkg_comp.sandbox_conffile
 58 | #         Path to a sandboxctl(8) configuration file to create a clean sandbox
 59 | #         for the current operating system.
 60 | 
 61 | 
 62 | # Controls sharding for the long-running integration tests in this file.
 63 | #
 64 | # Travis CI imposes a 1-hour limit for the execution of each individual build,
 65 | # which is insufficient to run all of our integration tests.  To make testing
 66 | # possible, we support sharding execution.  The INTTEST_SHARD variable
 67 | # indicates the identifier of this shard, which has to be in the range from
 68 | # zero to INTTEST_SHARDS.
 69 | #
 70 | # TODO(jmmv): This feature should be supported natively by Kyua.
 71 | : ${INTTEST_SHARD=0}
 72 | : ${INTTEST_SHARDS=1}
 73 | 
 74 | 
 75 | # Paths to installed files.
 76 | #
 77 | # Can be overriden for test purposes only.
 78 | : ${PKG_COMP_ETCDIR="__PKG_COMP_ETCDIR__"}
 79 | : ${PKG_COMP_SHAREDIR="__PKG_COMP_SHAREDIR__"}
 80 | 
 81 | 
 82 | integration_test_case() {
 83 |     atf_test_case "${1}" cleanup
 84 |     eval "${1}_inthead() { true; }"
 85 |     eval "${1}_head() { integration_head; ${1}_inthead; }"
 86 |     eval "${1}_body() { integration_body; ${1}_intbody; }"
 87 |     eval "${1}_cleanup() { integration_cleanup; }"
 88 | }
 89 | integration_head() {
 90 |     atf_set require.config "distdir pkgsrcdir sandbox_conffile"
 91 |     atf_set require.user "root"
 92 |     atf_set timeout 3600
 93 | }
 94 | integration_body() {
 95 |     cat >pkg_comp.conf <<EOF
 96 | DISTDIR="$(atf_config_get distdir)"
 97 | PACKAGES="$(pwd)/packages/pkg"
 98 | PBULK_LOG="$(pwd)/bulklog"
 99 | PBULK_PACKAGES="$(pwd)/packages/pbulk"
100 | PKGSRCDIR="$(atf_config_get pkgsrcdir)"
101 | SANDBOX_CONFFILE="$(pwd)/sandbox.conf"
102 | 
103 | LOCALBASE=/test/prefix
104 | PKG_DBDIR=/test/pkgdb
105 | SYSCONFDIR=/test/etc
106 | VARBASE=/test/var
107 | EOF
108 | 
109 |     cp "$(atf_config_get sandbox_conffile)" sandbox.conf
110 |     cat >>sandbox.conf <<EOF
111 | SANDBOX_ROOT="$(pwd)/sandbox"
112 | EOF
113 | 
114 |     mkdir packages
115 | }
116 | integration_cleanup() {
117 |     if [ -d sandbox ]; then
118 |         pkg_comp -c pkg_comp.conf sandbox-unmount -f -f || true
119 |         pkg_comp -c pkg_comp.conf sandbox-destroy || true
120 |     fi
121 | }
122 | 
123 | 
124 | # Configures the test to use Git to pull from pkgsrcgit.
125 | setup_fetch_from_local_git() {
126 |     [ -d "$(atf_config_get pkgsrcdir)" ] || atf_skip "pkgsrcdir does not" \
127 |         "point to a Git repository"
128 | 
129 |     echo "PKGSRCDIR='$(pwd)/pkgsrc'" >>pkg_comp.conf
130 |     # Prefer git for a faster test.
131 |     if which git >/dev/null 2>&1; then
132 |         echo "FETCH_VCS=git" >>pkg_comp.conf
133 |         echo "GIT_URL='file://$(atf_config_get pkgsrcdir)'" >>pkg_comp.conf
134 |     fi
135 | 
136 |     # Configure Git in case our tests want to modify the pkgsrc files.  In those
137 |     # cases, pkg_comp will need to stash the modifications during "fetch" and
138 |     # we need these details.
139 |     git config --global user.email "travis@example.com"
140 |     git config --global user.name "Travis"
141 | }
142 | 
143 | 
144 | # Asks the test to reuse existing binary packages, if allowed by the user.
145 | #
146 | # The binary packages are taken from the PACKAGES directory configured by the
147 | # user if they exist, and they must have been built by this test suite.  Note
148 | # that reusing binaries can lead to flakiness... but sometimes speedier tests
149 | # are desired.
150 | #
151 | # \param ... Basenames of the packages to reuse, if they exist.
152 | reuse_packages() {
153 |     if atf_config_has packages; then
154 |         echo "The 'packages' configuration variable is set."
155 |         echo "Reusing any pre-existing binary packages for: ${*}"
156 |         echo "Note: cp errors are normal here."
157 | 
158 |         mkdir -p packages/pkg/All
159 |         for pkgbase in "${@}"; do
160 |             cp "$(atf_config_get packages)/pkg/All/${pkgbase}"-[0-9]*.tgz \
161 |                packages/pkg/All || true
162 |         done
163 |     fi
164 | }
165 | 
166 | 
167 | # Asks the test to reuse existing bootstrap binaries, if allowed by the user.
168 | #
169 | # The bootstrap kits and packages are taken from the PACKAGES directory
170 | # configured by the user if they exist, and they must have been built by this
171 | # test suite.  Note that reusing binaries can lead to flakiness... but speedier
172 | # tests are preferrable during development.
173 | reuse_bootstrap() {
174 |     if atf_config_has packages; then
175 |         local packages="$(atf_config_get packages)"
176 | 
177 |         echo "The 'packages' configuration variable is set."
178 |         echo "Reusing any pre-existing bootstrap binary kits and packages."
179 |         echo "Note: cp errors are normal here."
180 | 
181 |         mkdir -p packages/pkg
182 |         cp "${packages}/pkg/bootstrap.tgz" packages/pkg || true
183 |         cp -rf "${packages}/pbulk" packages || true
184 | 
185 |         # Reuse the list of packages built for pbulk as the "bootstrap" packages
186 |         # to copy from the pkg tree.  This is an overapproximation though: the
187 |         # pbulk packages contain more than just the bootstrap packages, but the
188 |         # extra ones should not exist for our test scenarios.
189 |         local pbulk_packages="$(cd "${packages}/pbulk/All" \
190 |             && ls -1 *.tgz | sed -e 's,-[0-9].*$,,')"
191 |         reuse_packages ${pbulk_packages}
192 |     fi
193 | }
194 | 
195 | 
196 | # Persists bootstrap kits and built packages for future test reuse.
197 | #
198 | # If the user has allowed the reuse of built results across test invocations by
199 | # setting the 'packages' variable, copy all artifacts into PACKAGES so that
200 | # reuse_packages and reuse_bootstrap can use them later.
201 | save_state() {
202 |     if atf_config_has packages; then
203 |         local packages="$(atf_config_get packages)"
204 |         mkdir -p "${packages}/pbulk/All"
205 |         cp packages/pbulk/bootstrap.tgz "${packages}/pbulk" || true
206 |         cp packages/pbulk/All/* "${packages}/pbulk/All" || true
207 |         mkdir -p "${packages}/pkg/All"
208 |         cp packages/pkg/bootstrap.tgz "${packages}/pkg" || true
209 |         cp packages/pkg/All/* "${packages}/pkg/All" || true
210 |     fi
211 | }
212 | 
213 | 
214 | # Ensures that the given list of files exist.
215 | check_files() {
216 |     for file in "${@}"; do
217 |         [ -e "${file}" ] || atf_fail "Expected file ${file} not found"
218 |     done
219 | }
220 | 
221 | 
222 | # Ensures that the given list of files do not exist.
223 | check_not_files() {
224 |     for file in "${@}"; do
225 |         [ ! -e "${file}" ] || atf_fail "Unexpected file ${file} found"
226 |     done
227 | }
228 | 
229 | 
230 | # Ensures that a given package name appears in the pkg_summary files.
231 | #
232 | # \param dir Path to the directory containing the pkg_summary files.
233 | # \param pkgname Basename of the package to search for.
234 | check_pkg_summary() {
235 |     local dir="${1}"; shift
236 |     local pkgname="${1}"; shift
237 | 
238 |     gunzip -c "${dir}/pkg_summary.gz" | grep "PKGNAME=${pkgname}-[0-9]" \
239 |         || atf_fail "pkg_summary.gz does not contain ${pkgname}"
240 |     bunzip2 -c "${dir}/pkg_summary.bz2" | grep "PKGNAME=${pkgname}-[0-9]" \
241 |         || atf_fail "pkg_summary.bz2 does not contain ${pkgname}"
242 | }
243 | 
244 | 
245 | integration_test_case bootstrap_workflow
246 | bootstrap_workflow_intbody() {
247 |     atf_check -e ignore pkg_comp -c pkg_comp.conf sandbox-create
248 | 
249 |     # Bootstrap with an empty PACKAGES directory, which requires a rebuild from
250 |     # sources and results in binaries for reinstallation.
251 | 
252 |     atf_check \
253 |         -o match:'=> No binary package found for .*pbulk' \
254 |         -e match:'^pkg_comp: .*Bootstrapping pkg tools' \
255 |         -e match:'^pkg_comp: .*Bootstrapping pbulk tools' \
256 |         -e not-match:'^pkg_comp: .*Setting up .*using binary kit' \
257 |         pkg_comp -c pkg_comp.conf bootstrap
258 | 
259 |     # Now check that the contents of the sandbox are sane and store the list of
260 |     # files for later comparison with rebuilt sandboxes.
261 | 
262 |     check_files \
263 |         packages/pbulk/All/bmake-[0-9]* \
264 |         packages/pbulk/All/pbulk-[0-9]* \
265 |         packages/pbulk/All/pkg_install-[0-9]* \
266 |         packages/pbulk/bootstrap.tgz \
267 |         packages/pkg/All/bmake-[0-9]* \
268 |         packages/pkg/All/pkg_install-[0-9]* \
269 |         packages/pkg/bootstrap.tgz \
270 |         sandbox/test/etc/mk.conf \
271 |         sandbox/test/pkgdb/pkgdb.byfile.db \
272 |         sandbox/test/prefix/bin/bmake
273 |     rm -rf sandbox/pkg_comp/work sandbox/tmp/* sandbox/var/tmp/*
274 |     find sandbox | sort >exp_contents.txt
275 | 
276 |     # Bootstrap without destroying the sandbox, which should not do anything.
277 | 
278 |     atf_check pkg_comp -c pkg_comp.conf bootstrap
279 |     rm -rf sandbox/pkg_comp/work sandbox/tmp/* sandbox/var/tmp/*
280 |     atf_check -o file:exp_contents.txt -x "find sandbox | sort"
281 | 
282 |     # Bootstrap after destroying the sandbox but keeping PACKAGES, which should
283 |     # result in no rebuilds.
284 | 
285 |     atf_check -e ignore pkg_comp -c pkg_comp.conf sandbox-destroy
286 |     atf_check -e ignore pkg_comp -c pkg_comp.conf sandbox-create
287 | 
288 |     atf_check \
289 |         -o not-match:'=> No binary package found for .*pbulk' \
290 |         -o not-match:'=> Building for .*pbulk' \
291 |         -e match:'^pkg_comp: .*Bootstrapping pkg tools' \
292 |         -e match:'^pkg_comp: .*Bootstrapping pbulk tools' \
293 |         -e match:'^pkg_comp: .*Setting up .*/test/prefix using binary kit' \
294 |         -e match:'^pkg_comp: .*Setting up .*/pkg_comp/pbulk using binary kit' \
295 |         pkg_comp -c pkg_comp.conf bootstrap
296 |     rm -rf sandbox/pkg_comp/work sandbox/tmp/* sandbox/var/tmp/*
297 |     # When we build pbulk before, pkgsrc might have installed helper build
298 |     # tools.  We are now using binary packages, so those build tools will not be
299 |     # present.  Special-case them here, though this is quite fragile.
300 |     grep -v nbpatch exp_contents.txt >exp_contents.clean.txt
301 |     atf_check -o file:exp_contents.clean.txt -x "find sandbox | sort"
302 | 
303 |     # Ensure the public bootstrap kit does not leak pkg_comp settings.
304 |     tar xzvf packages/pkg/bootstrap.tgz ./test/etc/mk.conf
305 |     if grep /pkg_comp/ ./test/etc/mk.conf; then
306 |         atf_fail "Found internal paths to the sandbox in mk.conf"
307 |     fi
308 | 
309 |     atf_check -e ignore pkg_comp -c pkg_comp.conf sandbox-destroy
310 |     save_state
311 | }
312 | 
313 | 
314 | integration_test_case functional_pkgsrc_after_bootstrap
315 | functional_pkgsrc_after_bootstrap_intbody() {
316 |     reuse_bootstrap
317 | 
318 |     # Verify that the bootstrapped sandbox is sane and can build a simple
319 |     # package.
320 | 
321 |     atf_check -e ignore pkg_comp -c pkg_comp.conf sandbox-create
322 |     atf_check -o ignore -e ignore pkg_comp -c pkg_comp.conf bootstrap
323 | 
324 |     [ ! -e sandbox/test/prefix/bin/digest ] || atf_fail "digest was already" \
325 |         "installed in the sandbox but it should not be there"
326 |     rm -f packages/pkg/All/digest-[0-9]*
327 |     atf_check \
328 |         -o match:'=> Building for .*digest' \
329 |         -e ignore \
330 |         pkg_comp -c pkg_comp.conf sandbox-run \
331 |         /bin/sh -c 'cd /pkg_comp/pkgsrc/pkgtools/digest \
332 |             && /test/prefix/bin/bmake bin-install'
333 |     check_files \
334 |         packages/pkg/All/digest-[0-9]* \
335 |         sandbox/test/prefix/bin/digest
336 | 
337 |     # Rebootstrap the sandbox and try to reinstall the same package, which
338 |     # should result in a binary reuse.
339 | 
340 |     atf_check -e ignore pkg_comp -c pkg_comp.conf sandbox-destroy
341 |     atf_check -e ignore pkg_comp -c pkg_comp.conf sandbox-create
342 |     atf_check -o ignore -e ignore pkg_comp -c pkg_comp.conf bootstrap
343 | 
344 |     [ ! -e sandbox/test/prefix/bin/digest ] || atf_fail "digest was already" \
345 |         "installed in the sandbox but it should not be there"
346 |     atf_check \
347 |         -o not-match:'=> Building for .*digest' \
348 |         -e ignore \
349 |         pkg_comp -c pkg_comp.conf sandbox-run \
350 |         /bin/sh -c 'cd /pkg_comp/pkgsrc/pkgtools/digest \
351 |             && /test/prefix/bin/bmake bin-install'
352 |     check_files \
353 |         packages/pkg/All/digest-[0-9]* \
354 |         sandbox/test/prefix/bin/digest
355 | 
356 |     atf_check -e ignore pkg_comp -c pkg_comp.conf sandbox-destroy
357 |     save_state
358 | }
359 | 
360 | 
361 | integration_test_case build_workflow
362 | build_workflow_intbody() {
363 |     reuse_bootstrap
364 |     reuse_packages cwrappers digest pkgconf shtk sysbuild
365 | 
366 |     # Disable tests to make the build below a bit faster, and also to ensure
367 |     # that the extra mk.conf fragment is picked up.
368 |     echo "PKG_DEFAULT_OPTIONS=-tests" >extra.mk.conf
369 |     echo "EXTRA_MKCONF='$(pwd)/extra.mk.conf'" >>pkg_comp.conf
370 | 
371 |     # Verify that package name validation works.
372 |     atf_check \
373 |         -s exit:1 \
374 |         -o not-match:'Starting build' \
375 |         -e match:'W:.*foo.* does not exist' \
376 |         -e not-match:'W:.*sysbuild.* does not exist' \
377 |         -e match:'W:.*bar.* does not exist' \
378 |         -e not-match:'W:.*[^/]tmux.* does not exist' \
379 |         -e match:'W:.*misc/baz.* does not exist' \
380 |         pkg_comp -c pkg_comp.conf build foo sysbuild bar misc/tmux misc/baz
381 | 
382 |     # Start by building an arbitrary package that is certainly not included in
383 |     # the bootstrapping process.  The package we choose, sysbuild, has one
384 |     # dependency and we need at least one for our testing purposes.
385 |     atf_check -e ignore pkg_comp -c pkg_comp.conf sandbox-create
386 |     atf_check \
387 |         -o not-match:'Starting build of .*atf-[0-9]' \
388 |         -o match:'Starting build of .*shtk-[0-9]' \
389 |         -o match:'Successfully built .*shtk-[0-9]' \
390 |         -o match:'Starting build of .*sysbuild-[0-9]' \
391 |         -o match:'Successfully built .*sysbuild-[0-9]' \
392 |         -e ignore \
393 |         pkg_comp -c pkg_comp.conf build sysbuild
394 |     check_files packages/pkg/All/shtk-[0-9]*
395 |     check_files packages/pkg/All/sysbuild-[0-9]*
396 |     check_pkg_summary packages/pkg/All shtk
397 |     check_pkg_summary packages/pkg/All sysbuild
398 | 
399 |     # pbulk installs only intermediate dependencies but does not install the
400 |     # final package we asked, so install it now to ensure things work.
401 |     atf_check -o ignore -e ignore \
402 |         pkg_comp -c pkg_comp.conf sandbox-run \
403 |         /bin/sh -c 'cd /pkg_comp/packages/pkg/All && \
404 |             /test/prefix/sbin/pkg_add sysbuild'
405 |     check_files sandbox/test/prefix/bin/sysbuild
406 | 
407 |     # Delete an intermediate package and ensure it gets rebuilt even if not
408 |     # directly specified on the command line.
409 |     rm packages/pkg/All/shtk-*
410 |     check_not_files packages/pkg/All/shtk-[0-9]*
411 |     atf_check -o ignore -e ignore pkg_comp -c pkg_comp.conf build sysbuild
412 |     check_files packages/pkg/All/shtk-[0-9]*
413 |     check_files packages/pkg/All/sysbuild-[0-9]*
414 |     check_pkg_summary packages/pkg/All shtk
415 |     check_pkg_summary packages/pkg/All sysbuild
416 | 
417 |     # Delete an intermediate package and ensure that building it alone does
418 |     # not cause other binary packages to be discarded.
419 |     rm packages/pkg/All/shtk-*
420 |     check_not_files packages/pkg/All/shtk-[0-9]*
421 |     atf_check \
422 |         -o match:'Starting build of .*shtk-[0-9]' \
423 |         -o match:'Successfully built .*shtk-[0-9]' \
424 |         -o not-match:'Starting build of .*sysbuild-[0-9]' \
425 |         -e ignore \
426 |         pkg_comp -c pkg_comp.conf build shtk
427 |     check_files packages/pkg/All/shtk-[0-9]*
428 |     check_files packages/pkg/All/sysbuild-[0-9]*  # Has to be there!
429 |     check_pkg_summary packages/pkg/All shtk
430 |     check_pkg_summary packages/pkg/All sysbuild  # Has to be there!
431 | 
432 |     atf_check -e ignore pkg_comp -c pkg_comp.conf sandbox-destroy
433 |     save_state
434 | }
435 | 
436 | 
437 | integration_test_case auto_workflow
438 | auto_workflow_intbody() {
439 |     reuse_bootstrap
440 |     reuse_packages cwrappers digest pkgconf shtk sysbuild
441 | 
442 |     # Configure cvs to later verify that calls are properly invoked.
443 |     mkdir bin
444 |     cat >bin/cvs <<EOF
445 | #! /bin/sh
446 | touch $(pwd)/cvs.done
447 | exit 0
448 | EOF
449 |     chmod +x bin/cvs
450 |     export PATH="$(pwd)/bin:${PATH}"
451 |     echo "CVS_ROOT=fake-cvsroot" >>pkg_comp.conf
452 |     echo "CVS_TAG=fake-cvstag" >>pkg_comp.conf
453 | 
454 |     # Disable tests to make the build below a bit faster, and also to ensure
455 |     # that the extra mk.conf fragment is picked up.
456 |     echo "PKG_DEFAULT_OPTIONS=-tests" >extra.mk.conf
457 |     echo "EXTRA_MKCONF='$(pwd)/extra.mk.conf'" >>pkg_comp.conf
458 | 
459 |     # Verify that we must have provided at least one package before doing
460 |     # anything.
461 |     atf_check -s exit:1 -e match:"requires.*one package name" \
462 |         pkg_comp -c pkg_comp.conf auto
463 |     check_not_files cvs.done sandbox
464 | 
465 |     # Verify that package name validation works and that it happens before
466 |     # building anything.
467 |     atf_check \
468 |         -s exit:1 \
469 |         -o not-match:'Starting build' \
470 |         -e match:'W:.*foo.* does not exist' \
471 |         -e not-match:'W:.*sysbuild.* does not exist' \
472 |         -e match:'W:.*bar.* does not exist' \
473 |         -e not-match:'W:.*[^/]tmux.* does not exist' \
474 |         -e match:'W:.*misc/baz.* does not exist' \
475 |         pkg_comp -c pkg_comp.conf auto foo sysbuild bar misc/tmux misc/baz
476 |     check_files cvs.done
477 |     rm cvs.done
478 |     check_not_files sandbox
479 | 
480 |     # Start by building an arbitrary package that is certainly not included in
481 |     # the bootstrapping process.  The package we choose, sysbuild, has one
482 |     # dependency and we need at least one for our testing purposes.
483 |     echo "AUTO_PACKAGES='sysbuild'" >>pkg_comp.conf
484 |     atf_check \
485 |         -o not-match:'Starting build of .*atf-[0-9]' \
486 |         -o match:'Starting build of .*shtk-[0-9]' \
487 |         -o match:'Successfully built .*shtk-[0-9]' \
488 |         -o match:'Starting build of .*sysbuild-[0-9]' \
489 |         -o match:'Successfully built .*sysbuild-[0-9]' \
490 |         -e not-match:'pkg_comp: I: Updating pkgsrc tree' \
491 |         -e match:'pkg_comp: I: Creating sandbox' \
492 |         -e match:'pkg_comp: I: Destroying sandbox' \
493 |         pkg_comp -c pkg_comp.conf auto -f
494 |     check_files packages/pkg/All/shtk-[0-9]*
495 |     check_files packages/pkg/All/sysbuild-[0-9]*
496 |     check_not_files cvs.done sandbox
497 |     check_pkg_summary packages/pkg/All shtk
498 |     check_pkg_summary packages/pkg/All sysbuild
499 | 
500 |     # Now pass the packages to build on the command line.  As the sandbox has
501 |     # been recreated, we should not see the packages we asked earlier being
502 |     # built, but they should remain.
503 |     atf_check \
504 |         -o not-match:'Starting build of .*atf-[0-9]' \
505 |         -o match:'Starting build of .*shtk-[0-9]' \
506 |         -o match:'Successfully built .*shtk-[0-9]' \
507 |         -o not-match:'Starting build of .*sysbuild-[0-9]' \
508 |         -o not-match:'Successfully built .*sysbuild-[0-9]' \
509 |         -e not-match:'pkg_comp: I: Updating pkgsrc tree' \
510 |         -e match:'pkg_comp: I: Creating sandbox' \
511 |         -e match:'pkg_comp: I: Destroying sandbox' \
512 |         pkg_comp -c pkg_comp.conf auto -f shtk
513 |     check_files packages/pkg/All/shtk-[0-9]*
514 |     check_files packages/pkg/All/sysbuild-[0-9]*  # Has to be there!
515 |     check_pkg_summary packages/pkg/All shtk
516 |     check_pkg_summary packages/pkg/All sysbuild  # Has to be there!
517 |     check_not_files cvs.done sandbox
518 | 
519 |     # Check cvs operations in non-fast mode.
520 |     atf_check \
521 |         -o match:'Starting build of .*shtk-[0-9]' \
522 |         -o match:'Successfully built .*shtk-[0-9]' \
523 |         -e match:"Running.*cvs.*in $(atf_config_get pkgsrcdir)" \
524 |         -e match:'cvs.*-dfake-cvsroot.*update.*-d.*-P.*-rfake-cvstag' \
525 |         -e match:'pkg_comp: I: Updating pkgsrc tree' \
526 |         -e match:'pkg_comp: I: Creating sandbox' \
527 |         -e match:'pkg_comp: I: Destroying sandbox' \
528 |         pkg_comp -c pkg_comp.conf auto shtk
529 |     check_files cvs.done
530 | 
531 |     save_state
532 | }
533 | 
534 | 
535 | integration_test_case fetch_workflow
536 | fetch_workflow_inthead() {
537 |     atf_set require.progs "git"
538 | }
539 | fetch_workflow_intbody() {
540 |     setup_fetch_from_local_git
541 | 
542 |     check_not_files pkgsrc/mk/bsd.pkg.mk
543 |     atf_check \
544 |         -o ignore \
545 |         -e match:'pkg_comp: I: Updating pkgsrc tree' \
546 |         pkg_comp -c pkg_comp.conf fetch
547 |     check_files pkgsrc/mk/bsd.pkg.mk
548 | }
549 | 
550 | 
551 | integration_test_case auto_workflow_with_fetch
552 | auto_workflow_with_fetch_inthead() {
553 |     atf_set require.progs "git"
554 | }
555 | auto_workflow_with_fetch_intbody() {
556 |     reuse_bootstrap
557 |     reuse_packages cwrappers digest pkgconf shtk sysbuild
558 | 
559 |     setup_fetch_from_local_git
560 | 
561 |     # Disable tests to make the build below a bit faster, and also to ensure
562 |     # that the extra mk.conf fragment is picked up.
563 |     echo "PKG_DEFAULT_OPTIONS=-tests" >extra.mk.conf
564 |     echo "EXTRA_MKCONF='$(pwd)/extra.mk.conf'" >>pkg_comp.conf
565 | 
566 |     # Build one package, which will need to fetch the pkgsrc tree.
567 |     atf_check \
568 |         -o not-match:'Starting build of .*atf-[0-9]' \
569 |         -o match:'Starting build of .*shtk-[0-9]' \
570 |         -o match:'Successfully built .*shtk-[0-9]' \
571 |         -e match:'pkg_comp: I: Updating pkgsrc tree' \
572 |         pkg_comp -c pkg_comp.conf auto shtk
573 |     check_files pkgsrc/mk/bsd.pkg.mk
574 |     check_files packages/pkg/All/shtk-[0-9]*
575 |     check_pkg_summary packages/pkg/All shtk
576 | 
577 |     save_state
578 | }
579 | 
580 | 
581 | integration_test_case auto_workflow_reusing_sandbox
582 | auto_workflow_reusing_sandbox_intbody() {
583 |     reuse_bootstrap
584 |     reuse_packages cwrappers digest pkgconf shtk
585 | 
586 |     # Disable tests to make the build below a bit faster.
587 |     echo "PKG_DEFAULT_OPTIONS=-tests" >extra.mk.conf
588 |     echo "EXTRA_MKCONF='$(pwd)/extra.mk.conf'" >>pkg_comp.conf
589 | 
590 |     atf_check -o ignore -e ignore pkg_comp -c pkg_comp.conf sandbox-create
591 | 
592 |     check_files sandbox
593 |     atf_check \
594 |         -o not-match:'Starting build of .*atf-[0-9]' \
595 |         -o match:'Starting build of .*shtk-[0-9]' \
596 |         -o match:'Successfully built .*shtk-[0-9]' \
597 |         -e not-match:'pkg_comp: I: Updating pkgsrc tree' \
598 |         -e match:'pkg_comp: W: Reusing existing sandbox' \
599 |         pkg_comp -c pkg_comp.conf auto -f shtk
600 |     check_files packages/pkg/All/shtk-[0-9]*
601 |     check_files sandbox
602 | 
603 |     atf_check -o ignore -e ignore pkg_comp -c pkg_comp.conf sandbox-destroy
604 | 
605 |     save_state
606 | }
607 | 
608 | 
609 | integration_test_case logs_workflow
610 | logs_workflow_intbody() {
611 |     reuse_bootstrap
612 |     reuse_packages cwrappers digest
613 | 
614 |     setup_fetch_from_local_git
615 | 
616 |     atf_check -o ignore -e ignore pkg_comp -c pkg_comp.conf fetch
617 | 
618 |     check_files pkgsrc/pkgtools/verifypc/Makefile
619 |     cat >>pkgsrc/pkgtools/verifypc/Makefile <<EOF
620 | pre-extract: always-fail
621 | always-fail: .PHONY
622 | 	false
623 | EOF
624 | 
625 |     atf_check \
626 |         -s exit:1 \
627 |         -o match:'Starting build of .*verifypc' \
628 |         -o match:'Failed to build.*verifypc' \
629 |         -e match:'Failed to build.*verifypc.*detailed logs' \
630 |         pkg_comp -c pkg_comp.conf auto verifypc
631 | 
632 |     # Check presence of pbulk summary logs.
633 |     test -f bulklog/report.txt || fail "report.txt not found in bulklog"
634 |     test -f bulklog/report.html || fail "report.txt not found in bulklog"
635 | 
636 |     # Check presence of package-specific logs.
637 |     test -f bulklog/verifypc*/failure || fail "verifypc-specific logs not found"
638 | 
639 |     save_state
640 | }
641 | 
642 | 
643 | integration_test_case pkg_developer_workflow
644 | pkg_developer_workflow_intbody() {
645 |     reuse_bootstrap
646 |     reuse_packages cwrappers digest
647 | 
648 |     setup_fetch_from_local_git
649 | 
650 |     echo "PKG_DEVELOPER=yes" >>pkg_comp.conf
651 |     atf_check -o ignore -e ignore pkg_comp -c pkg_comp.conf fetch
652 |     atf_check -o ignore -e ignore pkg_comp -c pkg_comp.conf sandbox-create
653 |     atf_check -o ignore -e ignore pkg_comp -c pkg_comp.conf bootstrap
654 |     atf_check -o ignore -e ignore pkg_comp -c pkg_comp.conf sandbox-run \
655 |         touch /pkg_comp/pkgsrc/pkg_comp.cookie
656 | 
657 |     test -f pkgsrc/pkg_comp.cookie || fail "File created in pkgsrc not found"
658 |     grep "PKG_DEVELOPER.*yes" sandbox/pkg_comp/pbulk.mk.conf \
659 |         || fail "PKG_DEVELOPER setting not found in pbulk.mk.conf"
660 |     grep "PKG_DEVELOPER.*yes" sandbox/pkg_comp/pkg.mk.conf \
661 |         || fail "PKG_DEVELOPER setting not found in pkg.mk.conf"
662 | 
663 |     atf_check -o ignore -e ignore pkg_comp -c pkg_comp.conf sandbox-destroy
664 |     save_state
665 | }
666 | 
667 | 
668 | atf_init_test_cases() {
669 |     local tests=
670 |     tests="${tests} auto_workflow"
671 |     tests="${tests} auto_workflow_reusing_sandbox"
672 |     tests="${tests} auto_workflow_with_fetch"
673 |     tests="${tests} bootstrap_workflow"
674 |     tests="${tests} build_workflow"
675 |     tests="${tests} fetch_workflow"
676 |     tests="${tests} functional_pkgsrc_after_bootstrap"
677 |     tests="${tests} logs_workflow"
678 |     tests="${tests} pkg_developer_workflow"
679 | 
680 |     local i=0
681 |     for t in ${tests}; do
682 |         if [ "${i}" -eq "${INTTEST_SHARD}" ]; then
683 |             atf_add_test_case "${t}"
684 |         fi
685 |         i=$(( (${i} + 1) % ${INTTEST_SHARDS} ))
686 |     done
687 | }
688 | 


--------------------------------------------------------------------------------