├── LICENSE ├── README.md ├── bin ├── jagen └── jagen-stage ├── doc ├── Build.md ├── BuildSystem.md ├── Clean.md ├── Completions.md ├── Image.md ├── Init.md ├── List.md ├── Rules.md ├── Rust.md ├── Source.md ├── Update.md └── UserGuide.md ├── env.sh ├── init ├── lib ├── pkg │ ├── android-ndk.lua │ ├── android-ndk~r17b.lua │ ├── android-sdk-tools.lua │ ├── android-sdk-tools.sh │ ├── android-sdk-tools~4333796.lua │ ├── android-standalone-arm.lua │ ├── gdb.lua │ ├── gdb.sh │ ├── gdbserver.lua │ ├── gdbserver~8.lua │ ├── gdb~8.lua │ ├── linaro-aarch64-5.3.lua │ ├── linaro-arm-4.8.lua │ ├── linaro-arm-4.9.lua │ ├── linaro-arm-5.4.lua │ ├── linaro-arm-6.4.lua │ ├── linaro-arm-7.2.lua │ ├── repo.lua │ ├── repo.sh │ ├── rustup.lua │ ├── sourcery-mips-2012.03.lua │ ├── spawn.lua │ ├── system-native-clang.lua │ └── system-native-gcc.lua └── rules.lua ├── misc └── bash_completion └── src ├── Chunk.lua ├── Command.lua ├── Config.lua ├── Context.lua ├── Engine.lua ├── Jagen.lua ├── Log.lua ├── Module.lua ├── Ninja.lua ├── Options.lua ├── Package.lua ├── Refresh.lua ├── Rule.lua ├── Script.lua ├── Source.lua ├── System.lua ├── Target.lua ├── cmd.sh ├── command ├── build.lua └── source.lua ├── common.lua ├── common.sh ├── compile.sh ├── configure.sh ├── cprint.sh ├── help.lua ├── image.sh ├── install.sh ├── patch.sh ├── stages.sh ├── toolchain.sh ├── toolchain_libs.txt ├── unpack.sh └── util.sh /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014-2021 Oleg Kolosov 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Jagen 2 | 3 | [![Join the chat at https://gitter.im/bazurbat/jagen](https://badges.gitter.im/bazurbat/jagen.svg)](https://gitter.im/bazurbat/jagen?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) 4 | 5 | Jagen is a utility tool that saves software engineers time. It combines selected features of 6 | workspace and package managers to automate common day-to-day development and integration tasks such 7 | as setting up the environment, downloading distribution archives and toolchains, managing patches, 8 | keeping source repositories up to date and rebuilding parts of the project as needed. 9 | 10 | It is designed to be a very lightweight alternative to 11 | [Repo](https://source.android.com/setup/develop/repo)/[GClient](https://www.chromium.org/developers/how-tos/depottools) 12 | and [Yocto](https://www.yoctoproject.org). 13 | 14 | ## Warning 15 | 16 | The current `master` branch is a complete rewrite of rule engine and many other internal changes 17 | which break backwards compatibility. While the top level package rule syntax is about the same the 18 | internal processing is different, some property names were changed, etc. Given the opportunity I've 19 | decided to drop the most of the historical baggage as well. The process is not yet complete, so 20 | expect breakage. The documentation about rules is now mostly obsolete. 21 | 22 | The latest stable version before the rewrite was moved to the `legacy` branch. 23 | 24 | ## Introduction 25 | 26 | - [Features](#features) 27 | - [Getting Started](#getting-started) 28 | - [Package Rules](#package-rules) 29 | - [Building](#building) 30 | 31 | ## Reference 32 | 33 | - [Initialization](doc/Init.md) 34 | - [Build](doc/Build.md) 35 | - [Clean](doc/Clean.md) 36 | - [Update](doc/Update.md) 37 | - [List Information](doc/List.md) 38 | - [Managing Sources](doc/Source.md) 39 | - [Filesystem images](doc/Image.md) 40 | - [Rust Support](doc/Rust.md) 41 | - [Rules](doc/Rules.md) 42 | - [Build System](doc/BuildSystem.md) 43 | - [Bash completions](doc/Completions.md) 44 | 45 | ## Features 46 | 47 | - Simple declarative rules to define packages. 48 | - Initialize a build root using a single command (fast onboarding). 49 | - Reproduce the same environment on CI and development systems. 50 | - Build or rebuild any stage of any package respecting dependencies. 51 | - Complex projects can consist of layers (similar to OE/Yocto but much more straightforward). 52 | - Out of the box support for common build systems: CMake, Autotools, Android Gradle. In many cases 53 | can handle packages using bare Make as well without additional configuration. 54 | - Built-in source management facilities (similar to Repo and GClient) supporting 55 | [Git](https://git-scm.com) and [Mercurial](https://www.mercurial-scm.org) (Hg). 56 | - Automatic downloading of distribution archives (from everything curl supports \+ special handling 57 | of Google Drive). 58 | - Designed for cross-compilation from the start. 59 | - Easy to add custom pre-built toolchains. 60 | - Supports using of multiple toolchains in the same build root. 61 | - Packages can dynamically export the environment or settings for other packages. 62 | - First-class [Rust](https://www.rust-lang.org/) language support. 63 | - Fully assisted Bash completion (fast and offers items relevant to the current project). 64 | - An extensive set of facilities to accommodate packages with custom build systems or special 65 | needs. 66 | 67 | ## Getting Started 68 | 69 | A directory which groups related source code, distribution archives, build artifacts and other 70 | auxiliary files, is called a "workspace", a "root directory" or a "build root" interchangeably. 71 | Build roots can consist of multiple layers defining a project's environment and components which are 72 | merged together according to rules to form a complete build system. 73 | 74 | To start using Jagen create a directory for the workspace and make it current: 75 | 76 | ``` 77 | mkdir jagen-root && cd jagen-root 78 | ``` 79 | 80 | Initialize this directory by piping Jagen's init script into the shell: 81 | 82 | ``` 83 | curl -fsSL https://git.io/fhyEM | sh 84 | ``` 85 | 86 | Create a `rules.lua` file and add your rules there. Use the generated `./jagen` script to run the 87 | build system. 88 | 89 | If you already have some layers prepared pass their URLs to the shell after the `--`. For example, 90 | to create a workspace preconfigured with a [tutorial layer](https://github.com/bazurbat/jagen-start) 91 | located at https://github.com/bazurbat/jagen-start use the command: 92 | 93 | ``` 94 | curl -fsSL https://git.io/fhyEM | sh -s -- https://github.com/bazurbat/jagen-start 95 | ``` 96 | 97 | See a list of defined packages: 98 | 99 | ``` 100 | ./jagen list packages 101 | ``` 102 | 103 | Run the build system: 104 | 105 | ``` 106 | ./jagen build 107 | ``` 108 | 109 | During the build, Jagen will download/clone sources and run packages configure/compile/install 110 | stages in order as needed until everything is done. You can find the results in a `host` 111 | subdirectory which is used as a default staging directory for packages using the system's native 112 | toolchain. 113 | 114 | To modify the list of layers or flags or refresh the generated environment for an existing workspace 115 | run the `init` script again with the appropriate arguments inside the workspace's directory. To 116 | guard against unexpected changes to the script in the master branch rerun it from the local copy 117 | which was cloned during the workspace creation: 118 | 119 | ``` 120 | ./.jagen/init [OPTIONS...] 121 | ``` 122 | 123 | For more complex configuration changes than just setting flags or layers edit the `config.sh` file 124 | manually. Do not rerun the `init` script in this case because it generates the default `config.sh`. 125 | 126 | To download rules from the tutorial layer for experiments: 127 | 128 | ``` 129 | curl -fsSL https://raw.githubusercontent.com/bazurbat/jagen-start/master/rules.lua > rules.lua 130 | ``` 131 | 132 | Edit the `rules.lua` file and rerun the `./jagen build` command to bring the workspace up to date. 133 | 134 | ## Package Rules 135 | 136 | Jagen generates a build system from declarative rules found in `rules.lua` files across the 137 | workspace's layers. An order is significant with the later rules appending and overriding preceding 138 | definitions. The `rules.lua` file in the workspace itself is processed last, so it is a usual place 139 | to define package exclusions and build type (release/debug) specific for the build root. In the 140 | simplest case, the workspace's `rules.lua` can define all the packages as well but for easier 141 | sharing of the resulting environment, it is recommended to put it into a separate Git repository and 142 | use init commands outlined above once the ruleset stabilizes. 143 | 144 | For an example, here are the contents of the `rules.lua` from the tutorial layer: 145 | 146 | ```lua 147 | package { 'nanomsg', 148 | source = { 149 | location = 'https://github.com/nanomsg/nanomsg.git', 150 | tag = '1.1.4' 151 | }, 152 | build = 'cmake' 153 | } 154 | 155 | package { 'googletest', 156 | source = { 157 | location = 'https://github.com/google/googletest.git', 158 | branch = 'v1.8.x' 159 | }, 160 | build = 'cmake' 161 | } 162 | 163 | package { 'hello-nanomsg', 164 | source = 'https://github.com/bazurbat/hello-nanomsg.git', 165 | build = 'cmake', 166 | requires = { 'nanomsg', 'googletest' } 167 | } 168 | 169 | package { 'sqlite', 170 | source = { 171 | location = 'https://www.sqlite.org/2018/sqlite-autoconf-3250200.tar.gz', 172 | sha1sum = 'aedfbdc14eb700099434d6a743135743cff47393' 173 | }, 174 | build = { 175 | type = 'gnu', 176 | options = { 177 | '--disable-editline', 178 | '--disable-threadsafe', 179 | '--disable-dynamic-extensions', 180 | '--disable-fts4', 181 | '--disable-fts5', 182 | '--disable-json1', 183 | '--disable-rtree', 184 | '--disable-static-shell' 185 | } 186 | } 187 | } 188 | 189 | package { 'hello-sqlite', 190 | source = 'https://github.com/bazurbat/hello-sqlite.git', 191 | build = 'cmake', 192 | requires = 'sqlite' 193 | } 194 | ``` 195 | 196 | The simplest rule has the form: 197 | 198 | ```lua 199 | package { 'mypackage' } 200 | ``` 201 | 202 | Which defines an empty package named "mypackage". To enable Jagen's source management facilities you 203 | need to specify the source location of the package by setting the `source` property, for example: 204 | 205 | ```lua 206 | package { 'mypackage', 207 | source = 'https://github.com/nanomsg/nanomsg.git' 208 | } 209 | ``` 210 | 211 | Run the `jagen source update mypackage` command and find the nanomsg sources in the `src/mypackage` 212 | subdirectory of the build root. So, to use Jagen as a workspace manager it is enough to define a few 213 | packages this way and issue `jagen source update` command periodically to update them. If no other 214 | source properties are set Jagen will clone the default branch ("master" most of the time in the case 215 | of Git). Perhaps more common case, especially for dependent repositories, is a need to clone a 216 | specific tag. This can be done by setting the `tag` property of the source: 217 | 218 | ```lua 219 | package { 'nanomsg', 220 | source = { 'https://github.com/nanomsg/nanomsg.git', 221 | tag = '1.1.4' 222 | } 223 | } 224 | ``` 225 | 226 | or, alternatively, using the non-shorthand syntax: 227 | 228 | ```lua 229 | package { 'nanomsg', 230 | source = { 231 | location = 'https://github.com/nanomsg/nanomsg.git', 232 | tag = '1.1.4' 233 | } 234 | } 235 | ``` 236 | 237 | When a tag is set Jagen will not update the source after the initial clone because tags are not 238 | supposed to change upstream. You can also set a specific revision in a similar manner: 239 | 240 | ```lua 241 | package { 'nanomsg', 242 | source = { 243 | location = 'https://github.com/nanomsg/nanomsg.git', 244 | rev = 'ef4123ff70c74b47b66ef066ecf88d1ed3750dc3' 245 | } 246 | } 247 | ``` 248 | 249 | Note that you need to specify a full hash of the commit. This is also true for Mercurial sources 250 | because revision numbers are repository-specific. 251 | 252 | Instead of fixing the source to a single revision you can set a branch: 253 | 254 | ```lua 255 | package { 'nanomsg', 256 | source = { 257 | location = 'https://github.com/nanomsg/nanomsg.git', 258 | branch = 'ws' 259 | } 260 | } 261 | ``` 262 | 263 | This way the `jagen source update` command will try to bring the specified branch up to date. This 264 | form is useful for tracking upstream development. 265 | 266 | ### Building 267 | 268 | In addition to the source management, Jagen includes extensive facilities to orchestrate the 269 | building of software. Its role is to prepare an environment, compile and install dependencies and 270 | call a package's own build system with the correct parameters. To enable the build support you need 271 | to specify the type of the build system of the package, like so: 272 | 273 | ```lua 274 | package { 'nanomsg', 275 | source = { 276 | location = 'https://github.com/nanomsg/nanomsg.git', 277 | tag = '1.1.4' 278 | }, 279 | build = 'cmake' 280 | } 281 | ``` 282 | 283 | Run `jagen build nanomsg` command to build it. By default, the building of the package also implies 284 | that it needs to be installed to be found by dependent packages. Given the rule above the `nanomsg` 285 | will be installed in the `host` subdirectory of the build root. 286 | 287 | Package dependencies can be specified with `requires` property: 288 | 289 | ```lua 290 | package { 'nanomsg', 291 | source = { 292 | location = 'https://github.com/nanomsg/nanomsg.git', 293 | tag = '1.1.4' 294 | }, 295 | build = 'cmake' 296 | } 297 | 298 | package { 'googletest', 299 | source = { 300 | location = 'https://github.com/google/googletest.git', 301 | branch = 'v1.8.x' 302 | }, 303 | build = 'cmake' 304 | } 305 | 306 | package { 'hello-nanomsg', 307 | source = 'https://github.com/bazurbat/hello-nanomsg.git', 308 | build = 'cmake', 309 | requires = { 'nanomsg', 'googletest' } 310 | } 311 | ``` 312 | 313 | Just run `jagen build`. Jagen will clone/update sources and build everything in order as needed. 314 | -------------------------------------------------------------------------------- /bin/jagen: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if [ "${jagen_root_dir-}" ]; then 4 | "$jagen_root_dir/jagen" "$@" 5 | fi 6 | -------------------------------------------------------------------------------- /bin/jagen-stage: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # shellcheck disable=1090,2034,2086,2154,2155 3 | 4 | pkg_stage=${1:?} 5 | pkg__include_script=${2:?} 6 | 7 | jagen_include_dir=$(dirname "$pkg__include_script") 8 | 9 | . "$pkg__include_script" || exit 10 | . "${jagen_include_dir:?}/jagen.export.sh" || exit 11 | . "${jagen_dir:?}/src/stages.sh" || exit 12 | 13 | pkg__target="${pkg_name}:${pkg_stage}${pkg_config:+:${pkg_config}}" 14 | pkg__stamp_file="${jagen_build_dir:?}/${pkg__target}" 15 | pkg__log_file="${jagen_log_dir:?}/${pkg__target}.log" 16 | pkg__stdout=3 17 | pkg__stderr=4 18 | 19 | # the source fd can only be a literal number 20 | exec 3>&1 4>&2 21 | : >"$pkg__log_file" 22 | 23 | pkg__on_exit() { 24 | # dash calls an exit handler before an int handler with the status 130 when 25 | # errexit is active, just ignore this 26 | [ $1 = EXIT -a $2 = 130 ] && return 27 | # do not allow a recursive call which can happen in dash and ash if the 28 | # handler is too slow and a user is too quick pressing CTRL-C multiple times 29 | trap '' INT 30 | # make sure we are in a known state, do not interrupt the handler on error 31 | set +e 32 | 33 | local err=$2 34 | 35 | pkg__download_cleanup 36 | 37 | # the int handler is called with the status 130 except in dash with errexit 38 | # where it is called with 0 after the first command of an exit handler 39 | if [ $err != 0 -a $err != 130 ]; then 40 | printf "${pkg__target}\t${pkg__log_file}\n" >>"$jagen__cmd_failed_targets_file" 41 | fi 42 | 43 | if [ -s "$pkg__log_file" ]; then 44 | printf "\\034" >>"$pkg__log_file" 45 | fi 46 | 47 | if [ $1 = INT ]; then 48 | # exit indicating an interrupt to a parent shell, Bash in particular 49 | # requires this to distiguish an error exit from a user interrupt 50 | trap INT && kill -INT $$ 51 | fi 52 | 53 | # when the shell is in fact dash and the INT trap was activated due to an 54 | # errexit condition, we will still reach here despite the kill above 55 | } 56 | 57 | # we need to trap both INT and EXIT to have an opportunity to workaround the 58 | # discrepancies of signal handling between different shells 59 | trap 'pkg__on_exit INT $?' INT 60 | trap 'pkg__on_exit EXIT $?' EXIT 61 | 62 | if [ "$jagen__cmd_verbose" ] 63 | then 64 | pkg__pipe=$(mktemp -u /tmp/jagen.XXXXXXXX) || exit 65 | mkfifo "$pkg__pipe" || exit 66 | # tee will exit by itself when this process will close all fds associated 67 | # with the pipe, this will happen on exit automatically 68 | tee <"$pkg__pipe" -a "$pkg__log_file" || exit & 69 | exec >"$pkg__pipe" 2>&1 || exit 70 | # unlink a filesystem object, the pipe will exist as long as open fds 71 | rm -f "$pkg__pipe" || exit 72 | elif [ "$jagen__cmd_quiet" ] || ! [ "$jagen__has_console" ] || 73 | ! in_list "$pkg__target" $jagen_cmd_targets 74 | then 75 | exec >>"$pkg__log_file" 2>&1 || exit 76 | fi 77 | 78 | pkg_run cd "${jagen_build_dir:?}" || exit 79 | 80 | for use in $pkg_uses; do 81 | include "${jagen_include_dir}/${use}.export.sh" || exit 82 | done; unset use 83 | 84 | pkg__env_script="${jagen_include_dir}/${pkg_ref}.env.sh" 85 | if [ -f "$pkg__env_script" ]; then 86 | include "$pkg__env_script" 87 | fi 88 | 89 | pkg__work_dir=$(eval echo "\$pkg_stage_${pkg_stage}_work_dir") 90 | : ${pkg__work_dir:=$jagen_build_dir} 91 | 92 | if ! [ -d "$pkg__work_dir" ]; then 93 | pkg_run mkdir -p "$pkg__work_dir" 94 | fi 95 | pkg_run cd "$pkg__work_dir" 96 | 97 | pkg__stage_function=$(eval echo "\$pkg_stage_${pkg_stage}_function") 98 | : ${pkg__stage_function:=$(eval echo "jagen_stage_${pkg_stage}")} 99 | pkg__stage_function=$(jagen_to_shell_name "$pkg__stage_function") 100 | 101 | if is_function $pkg__stage_function; then 102 | debug2 "executing stage function $pkg__stage_function" 103 | $pkg__stage_function 104 | fi 105 | 106 | # if is_function "$pkg__stage_function"; then 107 | # # this command should remain unchecked for errexit to be able to interrupt 108 | # # the function as expected 109 | # eval "$pkg__stage_function"; pkg__err=$? 110 | # # an error status always causes an exit regardless of errexit 111 | # [ $pkg__err = 0 ] || exit $pkg__err 112 | # fi 113 | 114 | touch "$pkg__stamp_file" 115 | -------------------------------------------------------------------------------- /doc/Build.md: -------------------------------------------------------------------------------- 1 | ## Building 2 | 3 | `jagen build [OPTION...] [PATTERN...] [--] [TOOL OPTIONS...]` 4 | 5 | Builds or rebuilds the specified targets. 6 | 7 | ### Options 8 | 9 | Option | Description 10 | ---------------------|------------ 11 | -h, --help | print this help message 12 | -m, --match | print expanded value of target patterns and exit 13 | -c, --clean | clean package's build directories before the build 14 | -a, --all | continue until everything is up to date 15 | -n, --no-rebuild | do not rebuild targets which are already up to date 16 | -p, --progress | print the output of build targets after completion 17 | -f, --follow | follow a build output for the specified targets only 18 | -F, --follow-all | follow all build output 19 | -q, --quiet | inhibit build output 20 | -y, --ignore-dirty | ignore dirty status of source directories 21 | -x, --ignore-exclude | do not skip excluded packages 22 | 23 | ### Synopsis 24 | 25 | The specified patterns are expanded and matching targets are rebuilt. Use the 26 | '--match' option to print the matches without building anything. 27 | 28 | Use the '--clean' option to remove the package's build directories before the 29 | start. It also causes the 'configure' stage of the affected packages to become 30 | out of date. 31 | 32 | Use the '--all' option to build everything out of date in the current project in 33 | addition to the specified targets. 34 | 35 | The '--no-rebuild' option causes the command to behave similarly to 'make': it 36 | ensures that targets are up to date instead of rebuilding them unconditionally. 37 | 38 | The '--progress' option prints the buffered output of all build targets as soon 39 | as they complete. 40 | 41 | The '--follow' option allows monitoring the output from build commands in real 42 | time. It shows the output only for targets specified for the current build 43 | command. Works best when used for a single package or dependent targets because 44 | when there are several package builds in progress their output will be 45 | intermixed. 46 | 47 | The '--follow-all' option shows the output from all currently running build 48 | commands at the same time. This includes the targets building as dependencies of 49 | the ones specified as build command arguments and all others not currently up to 50 | date if used in combination with '--all' option. 51 | 52 | With the '--quiet' option the output from commands is collected to logs but not 53 | displayed on the console during the execution unless the stage finished with an 54 | error. 55 | 56 | Arguments after '--' will be passed literally to the underlying build tool. The 57 | handling depends on the build tool in question but be aware of the case when 58 | multiple targets matched as the same arguments will be passed to all of their 59 | build commands which might have surprising results. 60 | 61 | It is best to use this functionality to quickly adjust the behaviour of a 62 | single compile or configure command, such as: 63 | 64 | jagen build nanomsg:compile:host -- -v 65 | 66 | to pass `-v` flag to Ninja to temporarily enable verbose mode, or, in the case 67 | of 'make' build type: 68 | 69 | jagen build someppkg:compile:host -- V=1 70 | 71 | Another possibility: 72 | 73 | jagen build nanomsg:configure:host -- -DSOME_FLAG=ON 74 | 75 | to enable CMake flag `SOME_FLAG` without going to the build directory. 76 | -------------------------------------------------------------------------------- /doc/BuildSystem.md: -------------------------------------------------------------------------------- 1 | ## Build System 2 | 3 | ### Stages 4 | 5 | Each package in the generated build system divided by a number of stages which are performed in 6 | order: 7 | 8 | * clean 9 | * update/unpack 10 | * patch 11 | * clean:config 12 | * configure:config 13 | * compile:config 14 | * install:config 15 | 16 | Those stages can be subdivided to "common" and "config-specific". Each package has at least the 17 | following common stages: "clean", "update" (for SCM type sources) or "unpack" (for other sources) 18 | and "patch". 19 | 20 | If some build type is set (`build.type` is not `nil`) and there are some configs defined for the 21 | package then "configure" and "compile" stages are added. Additionally, `install.type` is set from 22 | `build.type` unless overridden which causes "install" stage to be added. 23 | 24 | The "clean" stages return the package to the initial state by removing temporary build directories. 25 | Common stage deletes `$pkg_work_dir` (\/build/\) directory by default, 26 | config-specific clean stages remove `$pkg_build_dir` (\/build/\). The 27 | directories removed by "clean" stages can be changes by setting `build.clean` property. 28 | 29 | The "update" stage updates the source repository by pulling or cloning if the target directory is 30 | not exists. The "unpack" stage unpacks the distribution archive to the `$pkg_work_dir`. 31 | 32 | The "patch" stages applies patches and copies supplementary files if defined or can be overridden 33 | to perform custom steps to prepare the package for the build. 34 | 35 | The "configure" stage runs `configure` for "gnu" build type or CMake for "cmake" build type passing 36 | it the defined build options. 37 | 38 | The "compile" runs runs steps necessary for compilation, for most build types this just means 39 | "make" command. 40 | 41 | The "install" stage runs `make install` or equivalent for the defined install type. 42 | 43 | ### Configs 44 | 45 | There are 2 predefined package configs in Jagen: "host" and "target". The "host" packages are 46 | installed to the `host` subdirectory of the project root and are generally meant as helpers for 47 | building the "target" packages (such as code generators, debuggers and utilities). The "target" 48 | packages are meant for the target system, so they are cross-compiled using the chosen target 49 | toolchain and are installed to the `target` subdirectory of the project which can also serve as a 50 | staging area. 51 | 52 | To allow running "host" packages directly for the current project the `host/bin` is added to 53 | `$PATH` and `host/lib` is added to `$LD_LIBRARY_PATH` when you source the generated `env.sh` script 54 | in the project directory. 55 | 56 | Additionally, if no package rules with explicitly set config found but there are definitions 57 | without config an empty package with "host" config is automatically defined for each such rule. 58 | This facilitates the use case when you just need to compile a set of packages for the local system 59 | using the native toolchain. 60 | 61 | If the package is built or installed (has non empty `build.type` or `install.type` it will also 62 | always have a config. So, even if you have not written any rules with the config set the "host" 63 | will be added anyway and you will need to spell the stage by full name such as 64 | `package:compile:host` when specifying the targets to build. 65 | 66 | ### Logs 67 | 68 | All output from the build command is saved in `log` subdirectory of the project in files named 69 | after the target with `.log` appended, such as `log/zeromq:patch.log` for patch stage of a package 70 | "zeromq" or `log/zeromq:install:host.log` for "install" stage of the package "zeromq" for the 71 | "host" config. 72 | 73 | When you see that some target failed inspect those log files to see the commands output to diagnose 74 | the problem. 75 | 76 | ### Targets 77 | 78 | Targets are specified as '\:\:\'. Available package stages are filtered 79 | with the given expression. Omitted component means 'all'. For example: 80 | 81 | - utils - select all stages of the utils package 82 | - utils:install - select all utils install stages 83 | - utils::host - select all host utils stages 84 | - utils:compile:host - select only host utils compile stage 85 | - :build:host - select host build stages of all packages 86 | 87 | When a target is successfully built the stamp file is created in the build directory with the name: 88 | `::`. This file is used to determine if the target is up to date. Deleting it 89 | will cause the corresponding target to be rebuilt unconditionally the next time the build system 90 | runs. 91 | 92 | ## Build system internals 93 | 94 | _TODO: this section is out of date._ 95 | 96 | The build system is generated from a set of rules represented by tables 97 | (key-value pairs) which are found in `rules.lua` and `pkg/.lua` files 98 | across layer directories. Each rule defines some piece of information about a 99 | package: build stages, type, location of source directory and so on. Rules with 100 | the same name are considered as belonging to the same package but evaluated 101 | independently at the point of reference. So, mentioning the package in the 102 | "requires" list of a package which has config will produce different result from 103 | standalone "package" declaration. Also order of rules matters, both in the rules 104 | file and across layers. See the `package` function in `src/Package.lua` file for 105 | complete information about evaluation logic. 106 | 107 | Package dependencies and build commands are written to `build.ninja` file in 108 | the `build` directory, additional package specific environment goes into 109 | "include scripts" in the `include` directory. All code dealing with include 110 | script generation is in `src/Script.lua` file which can be used as a reference. 111 | Dependencies are resolved by touching specifically named files in the build 112 | directory after the command succeeded, see the generated `build.ninja` for 113 | details. 114 | 115 | At the core of the build system is a `jagen-stage` script. Given a package 116 | name, stage and config it finds and imports all necessary environment files, 117 | runs the stage script and returns its result. Default build stage scripts and 118 | utility functions are placed in a `src/pkg.sh` file which can be used as a 119 | reference. Every `pkg/.lua` file can have `pkg/.sh` backing file 120 | in the same directory which is included by `jagen-stage` during the build and 121 | can be used to override default stages or environment. 122 | 123 | Layers, build type and directory locations are set in 'config.sh' which is 124 | generated during project initialization. It is also included indirectly in 125 | every build command. 126 | -------------------------------------------------------------------------------- /doc/Clean.md: -------------------------------------------------------------------------------- 1 | ### Cleaning 2 | 3 | ``` 4 | Usage: jagen clean [package[:config]...] 5 | 6 | Deletes package build directories or all generated files and directories 7 | inside the current build root. 8 | 9 | OPTIONS 10 | 11 | -y, --ignore-dirty ignore dirty status of source directories 12 | -x, --ignore-exclude do not skip excluded packages 13 | 14 | SYNOPSIS 15 | 16 | There can be multiple arguments in the form of: or :. 17 | Build directories of given packages will be removed. If is supplied 18 | the corresponding build directory will be removed, if only is supplied 19 | all build directories will be removed. 20 | 21 | If no arguments are given, than everything in the current build root is 22 | cleaned up. The following directories are recursively deleted: 23 | 24 | jagen_build_dir 25 | jagen_include_dir 26 | jagen_log_dir 27 | 28 | Actual paths depend on configuration. After the deletion regenerates the 29 | build system using the 'jagen refresh' command. 30 | ``` 31 | -------------------------------------------------------------------------------- /doc/Completions.md: -------------------------------------------------------------------------------- 1 | ### Install Bash command autocompletion 2 | 3 | 1. Make sure you have Bash completion package installed. 4 | 5 | Use the following command on Ubuntu/Debian Linux: 6 | 7 | sudo apt-get install bash-completion 8 | 9 | 2. Source the included `bash_completion` file in your profile. 10 | 11 | Add the following line: 12 | 13 | source "/misc/bash_completion" 14 | 15 | either to: 16 | 17 | - `~/.bash_completion` file, which should work also on macOS with an older 18 | completion package for Bash 3.2 19 | - `~/.local/share/bash-completion/completions/jagen.bash` file to autoload 20 | the completions as needed, which is a feature of a newer Bash completion 21 | version 22 | 23 | Replace the `` with the path to the Jagen repository. 24 | 25 | 3. Source your profile again, relogin or open a new terminal window to apply 26 | the changes. 27 | -------------------------------------------------------------------------------- /doc/Image.md: -------------------------------------------------------------------------------- 1 | ### Manage filesystem images 2 | 3 | ``` 4 | Usage: jagen image 5 | 6 | Manages filesystem images. 7 | 8 | COMMANDS 9 | 10 | create Create images 11 | 12 | The 'create' command creates filesystem images according to configuration. 13 | Currently the only possible command is: 14 | 15 | jagen image create rootfs 16 | 17 | in 'hi-linux' build root which creates rootfs image. 18 | ``` 19 | -------------------------------------------------------------------------------- /doc/Init.md: -------------------------------------------------------------------------------- 1 | ## Initializing 2 | 3 | Use the `init` script in the root of Jagen repository to initialize current directory as a project. 4 | You can pipe it to Shell directly from GitHub: 5 | 6 | mkdir -p ~/src/jagen-project 7 | cd ~/src/jagen-project 8 | curl -fsSL https://raw.githubusercontent.com/bazurbat/jagen/master/init | sh 9 | 10 | This will clone Jagen sources to `.jagen` subdirectory and initialize `~/src/jagen-project` 11 | directory as the project. To pass parameters to `init` when piping to Shell use the form: 12 | 13 | curl ... | sh -s -- param1 param2 14 | 15 | Non option arguments will be added to `$jagen_layers`, URL arguments will be cloned to `layer` 16 | subdirectory and the resulting directory will be added to the layers list. For example: 17 | 18 | curl -fsSL https://raw.githubusercontent.com/bazurbat/jagen/master/init | sh -s -- -a ccache \ 19 | https://github.com/bazurbat/jagen-lib.git 20 | 21 | Will initialize the project with `ccache` flag and `./layer/jagen-lib` layer cloned from 22 | `https://github.com/bazurbat/jagen-lib.git` in a single command. Use this form for quick 23 | deployment. 24 | 25 | Alternatively, if you want to share Jagen repository between different projects, you can clone it 26 | separately and run the `init` script reaching out by relative path to the checked out Jagen source 27 | directory. Like so: 28 | 29 | cd ~/src 30 | git clone https://github.com/bazurbat/jagen.git 31 | mkdir jagen-project 32 | cd jagen-project 33 | ../jagen/init 34 | 35 | The project directory can also be called "build root" for a family of packages. 36 | 37 | Having shared Jagen sources allows you to build several projects using the same Jagen version which 38 | can be more convenient if you have a lot of projects or are following the master closely, while 39 | piping produces more self-contained project and makes it easier to move it around. 40 | 41 | ### Options 42 | 43 | Option | Description 44 | ------------|------------ 45 | -h, --help | show usage information 46 | -a, --flag | add the flag to jagen_flags 47 | -f, --force | force to initialize non-empty directory 48 | -L | add the directory to jagen_layer_path 49 | 50 | Flags change Jagen behaviour globally per project. The following flags are reserved by the core: 51 | 52 | - `ccache` — activate the usage of `ccache` for all C/C++ toolchain commands 53 | - `offline` — causes all operations requiring the network to fail 54 | 55 | Reinitializing an existing project is possible with `--force` option. It will regenerate the 56 | `config.sh` saving the previous as `config.sh.bak`. Copy your old settings from it manually if 57 | necessary. 58 | 59 | All non-option arguments are considered to be paths to layers which contribute rule definitions and 60 | environment overrides to the project. Absolute paths are used as is, relative paths are resolved 61 | against the project directory, unqualified paths (not starting from `/`, `./` or `../`) are tried 62 | with each entry from `$jagen_layer_path` in turn. If an argument is an URL it will be cloned using 63 | Git to the `./layer` subdirectory inside the project. 64 | 65 | ### Description 66 | 67 | The `init` script creates `config.sh`, `env.sh` and `jagen` files inside the project directory. 68 | 69 | The `config.sh` file contains parameters from the `init` command line (layers, flags) and other 70 | global settings. Edit it manually to adjust the parameters or global environment. It is sourced 71 | every time the build system runs. 72 | 73 | Source the `env.sh` script to initialize current Shell environment to work with the current project 74 | from now on. This will make the following changes: 75 | 76 | - Add Jagen's bin directory to `$PATH` so you can run `jagen` command from anywhere. 77 | - Add project's bin directory to `$PATH` to make generated toolchain wrapper scripts available. 78 | - Add `host/bin` to `$PATH` and `host/lib` to `$LD_LIBRARY_PATH` so you can use packages installed 79 | with "host" config. 80 | - Activate Bash completions for `jagen` command if installed. 81 | 82 | This is recommended mode for interactive work. Note that mixing the environments from different 83 | projects is not supported and will likely produce unexpected results. 84 | 85 | Use the `jagen` script inside the project directory to run Jagen commands for this project from 86 | outside without modifying the environment. This is useful for one shot tasks such as building from 87 | an IDE or CI. 88 | 89 | A layout for a typical project is the following: 90 | 91 | Path | Description 92 | ---------|------------ 93 | /bin | generated helper scripts 94 | /build | build system working directories 95 | /dist | the downloaded distribution archives and files 96 | /host | install root for 'host' configuration 97 | /include | package-specific include scripts generated from rules 98 | /layer | project-specific layers cloned by `init` 99 | /lib | project-specific rules and pkg scripts 100 | /log | build log files 101 | /src | checked out sources for SCM packages 102 | /target | install root for 'target' configuration 103 | 104 | The generated environment binds the project to the corresponding jagen source directory. If one or 105 | the other is moved or sourced from different root from which it was originally initialized (like 106 | chroot or Docker container) any build-related command will likely produce wrong results. 107 | -------------------------------------------------------------------------------- /doc/List.md: -------------------------------------------------------------------------------- 1 | ### Cleaning 2 | 3 | ``` 4 | Usage: jagen list [packages|layers] [OPTIONS...] 5 | 6 | Lists various information about the current project. 7 | 8 | COMMANDS 9 | 10 | packages List package rules and their origin 11 | 12 | The 'packages' command displays a list of all currently defined packages 13 | along with contexts where their definitions were found. Contexts could be 14 | rule files or other packages which mention given package as their requires. 15 | In the displayed filenames the parent directory of the current project is 16 | shown as '...'. 17 | 18 | packages options: 19 | 20 | --all, -a 21 | Show also implicit rules added by the generator such as the toolchain 22 | dependencies. These rules will be marked with "*". 23 | 24 | --depth,-d 25 | Set the maximum depth of the rule contexts displayed. If none was 26 | specified the 0 is the default which results in showing only the 27 | toplevel packages explicitly defined in the rule files. If the option 28 | is given without a value it is set to 999 which means show all 29 | contexts. 30 | 31 | layers Show currently defined layers and their file paths. 32 | ``` 33 | -------------------------------------------------------------------------------- /doc/Rust.md: -------------------------------------------------------------------------------- 1 | ## Rust 2 | 3 | Jagen supports packages written in [Rust](https://www.rust-lang.org) language including 4 | cross-compilation. It transparently setups `rustup`, installs toolchains and adds targets as 5 | needed. It adds convenience on top of bare `rustup` if you need to build multiple Rust packages for 6 | different target systems, express dependencies from native (C/C++) packages or quickly initialize 7 | CI environment. 8 | 9 | To enable Rust support for a package set its build type to `rust`: 10 | 11 | ```lua 12 | package { 'hello', 13 | build = 'rust' 14 | } 15 | ``` 16 | 17 | Note that this and further example package definitions do not contain `source` property. If this is 18 | the case Jagen expects to find the package sources in `$jagen_src_dir/hello` directory which is 19 | `/src/hello` by default. You can copy or link already existing Rust project into this 20 | directory or add `source` definition to the examples to let Jagen clone the sources from a remote 21 | location. See more about defining package sources in [Managing Sources](Source.md) section. 22 | 23 | By default `stable` Rust toolchain is assumed for the package which will be downloaded and cached 24 | in `$jagen_dist_dir/rustup` directory during the first build. To change the toolchain use the 25 | `rust_toolchain` build property: 26 | 27 | ```lua 28 | package { 'hello', 29 | build = { 30 | type = 'rust', 31 | rust_toolchain = '1.26.2' 32 | } 33 | } 34 | ``` 35 | 36 | A given name will be passed to `rustup toolchain install` command if the toolchain is not already 37 | installed so anything this command accepts can be used as the value. For each distinct 38 | `` value the package `rust-` will be declared and added to the 39 | project, for example: `rust-stable` (the default) or `rust-1.26.2`. You can use this package name 40 | to reinstall the toolchain manually but this normally is not needed because it will be pulled by 41 | `rust` packages as a dependency. 42 | 43 | When changing the toolchain for already compiled package a clean rebuild is necessary to recompile 44 | the package using the new toolchain, use `--clean` argument for the "build" command, for example: 45 | 46 | ``` 47 | jagen build --clean hello 48 | ``` 49 | 50 | To change the target system for the package use the `system` property, such as 51 | `x86_64-unknown-linux-musl` to have fully static binaries. 52 | 53 | ```lua 54 | package { 'hello', 55 | build = { 56 | type = 'rust', 57 | rust_toolchain = '1.26.2', 58 | system = 'x86_64-unknown-linux-musl' 59 | } 60 | } 61 | ``` 62 | 63 | You can specify any target from `rustup target list` but generally non-native targets will fail to 64 | link if the system compiler is not able handle the needed binary format. Set the `toolchain` 65 | property to specify the cross-compiler which will be used to link the target binaries. 66 | 67 | ```lua 68 | package { 'hello', 69 | build = { 70 | type = 'rust', 71 | toolchain = 'android-standalone-arm' 72 | } 73 | } 74 | ``` 75 | 76 | Note that the `system` property is not necessary in this case because it will be derived from the 77 | toolchain package. 78 | 79 | The definitions for the following toolchains are supplied with Jagen itself and can be used without 80 | additional configuration: 81 | 82 | android-standalone-arm 83 | linaro-aarch64-5.3 84 | linaro-arm-4.8 85 | linaro-arm-4.9 86 | linaro-arm-5.4 87 | linaro-arm-6.4 88 | linaro-arm-7.2 89 | sourcery-mips-2012.03 90 | 91 | The specified toolchain will be added to the package dependencies, downloaded and unpacked 92 | automatically during the build. 93 | 94 | For each distinct Rust toolchain name and target (specified as a `system` or derived from a 95 | toolchain) a separate `rust-*` package will be added to the project, such as: 96 | `rust-1.26.2-x86_64-unknown-linux-musl` or `rust-stable-android-standalone-arm`. This way you can have 97 | different packages using different Rust toolchains and targets in the same project. 98 | 99 | The `rust` type packages are built out of source with `$CARGO_TARGET_DIR` set to 100 | `$pkg_build_dir` which defaults to `$jagen_build_dir//` which will be 101 | `build/hello/host` directory in the current project for the examples above. 102 | 103 | Each project has a separate `$RUSTUP_HOME` (which defaults to `$jagen_dist_dir/rustup` and 104 | `$CARGO_HOME` (which defaults to `$jagen_dist_dir/cargo`). When the `rustup` package is 105 | installed it is linked to `host/bin` directory which is added to `$PATH` by default, so you can use 106 | `rustup` and other Cargo commands manually to manage Rust environment if the project-specific 107 | `env.sh` is sourced in the current Shell. 108 | -------------------------------------------------------------------------------- /doc/Source.md: -------------------------------------------------------------------------------- 1 | # Managing Sources 2 | 3 | Jagen provides an abstraction on top of Git, Hg and Repo SCM systems to allow working with them 4 | using the same set of commands. 5 | 6 | ## Rules 7 | 8 | To enable source management facilities add `source` property to the package rule: 9 | 10 | ```lua 11 | package { 'nanomsg', 12 | source = 'https://github.com/nanomsg/nanomsg.git' 13 | } 14 | ``` 15 | 16 | This is a shorthand form for the following expanded rule: 17 | 18 | ```lua 19 | package { 'nanomsg', 20 | source = { 21 | type = 'git', 22 | location = 'https://github.com/nanomsg/nanomsg.git' 23 | } 24 | } 25 | ``` 26 | 27 | If the source type was not specified then Jagen will apply a heuristic algorithm to try to 28 | determine it: if the location ending with well known archive extension then the type is set to 29 | `dist`, if it is ending with ".git" or starting with "git@" or "https://github.com" or matching Git 30 | style SSH URL then the type is `git`, if the location ends with ".hg" then the type is `hg`, if it 31 | equals to "." then the type is `dir`, otherwise it is `dist`. 32 | 33 | There is a shorthand form for overriding the type: 34 | ```lua 35 | source = { 'git', 'https://some/repo/location' } 36 | ``` 37 | 38 | But, in general, if you want to override some settings or specify a branch or tag you will need to 39 | use the expanded key-value form. 40 | 41 | The source types `git`, `hg` and `repo` are considered "SCM" sources and can be managed by `source` 42 | command. Other source types are handled implicitly during the "unpack" package build stage. 43 | 44 | See [Source](Rules.md#source) section about rules for the list and description of all supported 45 | properties. The `filename` and `basename` are set if the `location` is set. The `name` and `dir` 46 | are always set. 47 | 48 | Note that setting a revision or tag for Git source makes the working directory "detached" so it is 49 | fixed on a given reference. If a branch is specified Jagen will check it out locally if not already 50 | exists (using the default behaviour of `checkout` command) and try to follow by merging its remote 51 | with fast-forward on each update. This mode can also be convenient for repositories which are 52 | seldom worked on to allow making commits and pushing from the working directory the same way as 53 | when using Git manually. 54 | 55 | If neither rev nor tag nor bookmark nor branch is specified then no corresponding parameter will be 56 | passed to SCM command invocation. The SCM tool will choose it's own default. For Git this means 57 | that the "master" branch will be checked out. Additionally, the configuration will be set to update 58 | only "master" branch for shallow repositories or all branches for normal (not shallow) repositories 59 | during fetch the same way as `git clone` command initially does. For Hg sources the "default" 60 | branch will be chosen. 61 | 62 | As a safety measure, during the update stage or with the `jagen source update` command Jagen will 63 | refuse to update repositories which have unsaved changes (dirty). Clean them or stash the changes 64 | to let the update continue. 65 | 66 | For packages under an active development, when you want to have a local checkout of a branch which 67 | does not have the corresponding remote yet, or want to have complete control over the repository 68 | sources set the `exclude` property to `true`. This will make the default unpack and patch stages 69 | implementations skip the source directory during the build. This exclusion does not apply to `jagen 70 | source` command. 71 | 72 | ## Commands 73 | 74 | ``` 75 | jagen source [PACKAGES...] 76 | ``` 77 | 78 | An optional `PACKAGES` argument should be a list of SCM packages defined in the current 79 | environment. If nothing is specified then the command will be applied to all SCM packages. 80 | 81 | Command | Description 82 | --------|------------ 83 | status | Show packages location, head commit and dirty status 84 | update | Update the sources to the latest upstream version 85 | clean | Clean up packages source directories 86 | delete | Delete packages source directories 87 | dirty | Check if packages source directories have any changes 88 | each | Execute Shell command inside each source directory 89 | 90 | The `status` command prints SCM packages status in human readable form. 91 | 92 | The `update` command fetches the latest sources from upstream and tries to merge them. It 93 | does nothing if there are modifications in the working directory (dirty returns true); 94 | commit, stash or clean changes before issuing the 'update' command in this case. 95 | 96 | The `delete` command deletes packages source directories. 97 | 98 | The `dirty` command exits with 0 (true) status if any of the specified packages source 99 | directories have changes. It exits with 1 (false) status if all sources are clean. 100 | Intended for usage by shell scripts. 101 | 102 | It is possible to use `src` as an alternative command name instead of `source`. 103 | 104 | ## clean 105 | 106 | Resets the modifications to the HEAD state and deletes all extra files in the packages 107 | source directory. 108 | 109 | Long | Short | Description 110 | ---------------|-------|------------ 111 | ignore-dirty | y | force cleanup of dirty source directories 112 | ignore-exclude | Y | force cleanup of the sources which are excluded 113 | ignored | i | additionally delete files ignored by VCS when ignoring exclude 114 | 115 | The command will skip directories which have modified or untracked files (dirty) unless 116 | the `--ignore-dirty` option is specified. It will also skip the sources which are excluded 117 | via `jagen_source_exclude` setting or have `source.exclude` property set unless the 118 | `--ignore-exclude` option is specified. One of these options does not imply the other. To 119 | force a cleanup of a source which is excluded and dirty both should be specified at the 120 | same time. 121 | 122 | Files ignored by VCS are also deleted for normal sources but for the excluded sources only 123 | a default cleanup is performed. To really delete all files when ignoring exclude specify 124 | `--ignored` option in addition to the `--ignore-exclude`. 125 | 126 | ## each 127 | 128 | ``` 129 | jagen source each 130 | ``` 131 | 132 | Execute Shell command for each source directory. 133 | 134 | ### Synopsis 135 | 136 | The `each` subcommand concatenates its arguments and executes the result as 137 | Shell command inside the source directories. Use the `--type` argument to 138 | filter the source directories by type, e.g. `--type git` will run the command 139 | only for Git sources. If the type is not specified but the command starts 140 | with one of the known source types (git, hg and repo) then it will be set 141 | implicitly, otherwise the command will be run for all source directories. 142 | 143 | ### Examples 144 | 145 | Run `git status` for sources of type "git": 146 | 147 | jagen source each git status 148 | 149 | Run `ls` for all source directories: 150 | 151 | jagen source each ls 152 | 153 | Run `ls` only for "hg" sources: 154 | 155 | jagen source each --type hg ls 156 | -------------------------------------------------------------------------------- /doc/Update.md: -------------------------------------------------------------------------------- 1 | # Updating 2 | 3 | ``` 4 | jagen update [|jagen|self]... 5 | ``` 6 | 7 | Updates the specified layers or Jagen itself. 8 | 9 | ## Synopsis 10 | 11 | Specify a list of shell-like patterns of layer names to update. To see all currently defined layers 12 | use the `jagen list layers` command. If nothing is specified then all layers will be updated. 13 | 14 | Special names 'jagen' and 'self' can be added to the list to also update the Jagen repository 15 | associated with the project. These special names do not participate in the layer name pattern 16 | matching and should be specified explicitly. 17 | 18 | ## Examples 19 | 20 | Update all currently defined layers: 21 | 22 | jagen update 23 | 24 | Update only Jagen: 25 | 26 | jagen update self 27 | 28 | Update all layers and Jagen (note that you need to escape `*` from the Shell to pass it to Jagen): 29 | 30 | jagen update self '*' 31 | 32 | Update only layers with names starting with 'ab', containing 'cd` and ending with 'ef': 33 | 34 | jagen update 'ab*' '*cd*' '*ef' 35 | 36 | -------------------------------------------------------------------------------- /doc/UserGuide.md: -------------------------------------------------------------------------------- 1 | # User Guide 2 | 3 | _The guide is work in progress._ 4 | 5 | ## Introduction 6 | 7 | The workflow in Jagen is organized around projects (also called "build roots") which consist of a 8 | number of "layers". Each layer can provide rules, configuration, "pkg" files, patches and other 9 | resources which are used to generate a build system and environment of the project. This build 10 | system tracks packages dependencies, allows rebuilding of the parts or the whole project and 11 | managing of the source directories. 12 | 13 | Typically, you have project's environment (`env.sh`) sourced in a terminal and do all actions 14 | related to the project from this terminal window. This way you will have Shell command 15 | auto-completion working properly, toolchains and tools compiled for the host in you PATH so you can 16 | run them directly or perform some other actions manually. For non-interactive work (such as 17 | launching the build from an IDE) there is a `jagen` script in each project's directory which 18 | initializes the environment before redirecting the command to the Jagen's instance. Running this 19 | script does not change the current Shell environment. 20 | 21 | ## Requirements 22 | 23 | - [Lua](https://www.lua.org) 5.1 or 5.2 ([LuaJIT](http://luajit.org) 2.0 works as well) 24 | - [Ninja build](https://ninja-build.org) 25 | - [ccache](https://ccache.samba.org) is recommended if you plan to compile a lot of C/C++ code 26 | 27 | On Ubuntu Linux you can install the required packages using the command: 28 | 29 | sudo apt-get install lua5.1 ninja-build ccache 30 | 31 | ## Initialization 32 | 33 | Use the `init` script in the root of Jagen source directory to initialize the current directory as 34 | the project. It can be piped to Shell directly from GitHub: 35 | 36 | mkdir -p ~/src/jagen-project && cd ~/src/jagen-project 37 | curl -fsSL https://raw.githubusercontent.com/bazurbat/jagen/master/init | sh 38 | 39 | You can pass parameters such as layers and flags to init using the form: 40 | 41 | curl ... | sh -s -- param1 param2 42 | 43 | If you prefer to clone the repository manually to share the same Jagen instance between several 44 | projects you can run `init` locally after the checkout. See [Init](Init.md) reference for more 45 | details. 46 | 47 | Add the provided Bash completions file `.jagen/misc/bash_completion` to your Shell completion 48 | configuration. For one time trial just source this file in the current Shell. 49 | 50 | . .jagen/misc/bash_completion 51 | 52 | After that source the generated `env.sh` file as well. 53 | 54 | . ./env.sh 55 | 56 | Now the current Shell is initialized to work with the project and you can execute the `jagen` 57 | command from anywhere. 58 | 59 | ## Adding Packages 60 | 61 | The next step is to declare packages for the build system. The package declarations are found in 62 | "rule files" called `rules.lua` across the "jagen path". The "jagen path" always contains the `lib` 63 | directory of the current project as the last entry which means that it overrides the definitions 64 | coming from layers if any. 65 | 66 | Open the `lib/rules.lua` file. 67 | 68 | ```lua 69 | package { 'nanomsg', 70 | source = 'https://github.com/nanomsg/nanomsg.git', 71 | build = 'cmake' 72 | } 73 | ``` 74 | 75 | This rule defines a package named "nanomsg" which should be downloaded from 76 | https://github.com/nanomsg/nanomsg.git and built using CMake. 77 | 78 | See more about writing rules in [Rules](Rules.md) reference. 79 | 80 | ## Building 81 | 82 | Use `jagen build` command interact with the build system. 83 | 84 | Running `jagen build` without any arguments ensures that all targets are up to date. When there is 85 | nothing left to build you will see the: 86 | 87 | ninja: no work to do. 88 | 89 | message from ninja. To rebuild a target specify it as the build command argument. 90 | 91 | See more about building packages in [Build](Build.md) reference. 92 | 93 | ## Managing Sources 94 | 95 | Use `jagen source` command to manage sources. 96 | 97 | See more about managing package sources in [Source](Source.md) reference. 98 | 99 | ## Updating 100 | 101 | Update all layers in the current projects using the `jagen update` command. Update the associated 102 | Jagen repository using the `jagen update self` command. 103 | 104 | See more about update command in [Update](Update.md) reference. 105 | -------------------------------------------------------------------------------- /env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | jagen_S="$(printf '\n!')"; jagen_S=${jagen_S%!} 4 | jagen_FS="$(printf '\t')" 5 | jagen_IFS="$(printf '\n\t')" 6 | 7 | # These globals are coming from root's env.sh 8 | jagen_dir="${jagen_dir:?}" 9 | jagen_root_dir="$jagen_root_dir" 10 | 11 | jagen_lua="${jagen_lua-}" 12 | : ${jagen_lua:=$(command -v luajit)} 13 | : ${jagen_lua:=$(command -v lua)} 14 | 15 | jagen_debug="${jagen_debug-}" 16 | 17 | jagen__src_dir="$jagen_dir/src" 18 | jagen_root_lib_dir="$jagen_root_dir/lib" 19 | 20 | jagen_build_dir="$jagen_root_dir/build" 21 | jagen_include_dir="$jagen_build_dir/include" 22 | 23 | jagen_build_verbose=${jagen_build_verbose-} 24 | 25 | . "$jagen_dir/src/common.sh" || return 26 | 27 | jagen__set_path || return 28 | 29 | import env || true # it is OK if no env was found 30 | 31 | # export all jagen_* variables 32 | for var in $(set | LC_ALL=C sed -n 's/^\(jagen_[[:alnum:]][[:alnum:]_]*\)=.*/\1/p'); do 33 | export "$var" 34 | done; unset var 35 | -------------------------------------------------------------------------------- /init: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -ef 4 | 5 | jagen_url="https://github.com/bazurbat/jagen.git" 6 | env_file="env.sh" 7 | runner_script="jagen" 8 | 9 | NL=$(printf '\n!'); NL=${NL%!} 10 | 11 | show_usage() { 12 | cat </init [OPTIONS...] 23 | 24 | DESCRIPTION 25 | 26 | The script sets up the current working directory as Jagen workspace. 27 | When piped to Shell it will clone the Jagen source repository into the 28 | '.jagen' subdirectory. Alternatively, it can be run locally by reaching out 29 | with a relative path to an already checked out Jagen source repository. 30 | 31 | OPTIONS 32 | 33 | -h|--help show this usage information 34 | -a|--flag add an optional feature to the workspace 35 | -b|--branch clone the specified layers branch 36 | -j|--jagen-branch clone the specified Jagen branch 37 | --root force init as a build root 38 | --project force init as a project 39 | -L add a directory to the layers search path 40 | 41 | EOF 42 | } 43 | 44 | say() { 45 | echo "jagen-init: $*" 46 | } 47 | 48 | die() { 49 | say "error: $*" >&2 50 | exit 1 51 | } 52 | 53 | need_cmd() { 54 | if ! command -v "$1" >/dev/null; then 55 | die "could not find the '$1' command" 56 | fi 57 | } 58 | 59 | git_clone() { 60 | local url="$1" dir="$2" branch="$3" 61 | if ! [ -d "$dir" ]; then 62 | need_cmd git 63 | git clone --depth 1 ${branch:+--branch $branch} -- "$url" "$dir" 64 | fi 65 | } 66 | 67 | path_is_url() { 68 | [ "$1" != "${1#?*://?*/?*}" ] && return # 0 if match: prot://host/dir 69 | [ "$1" != "${1#*/?}" ] || return # 1 if not contains / or ends with / 70 | [ "$1" != "${1#git@?*:?*/?*}" ] && return # 0 if matches like git@github.com:name/dir 71 | [ "$1" != "${1#?*:?*/?*}" ] && return # 0 if matches like github.com:name/dir 72 | return 1 73 | } 74 | 75 | real_path() { 76 | if [ "$1" ]; then 77 | (cd "$1" && pwd) 78 | else 79 | pwd 80 | fi 81 | } 82 | 83 | is_build_root() { 84 | local dir="${1:-$PWD}" 85 | [ -d "$dir" ] && 86 | [ -f "$dir/$env_file" ] && 87 | [ -x "$dir/$runner_script" ] 88 | } 89 | 90 | parse_command_line() { 91 | while [ $# -gt 0 ]; do 92 | case $1 in 93 | -h|--help) 94 | show_usage; exit 0 ;; 95 | -a|--flag) 96 | [ "$2" ] || die "the --flag (-f) option requires an argument" 97 | jagen_flags="$jagen_flags $2"; shift ;; 98 | -b|--branch) 99 | [ "$2" ] || die "the --branch (-b) option requires a branch name argument" 100 | jagen_layer_branch="$2"; shift ;; 101 | -j|--jagen-branch) 102 | [ "$2" ] || die "the --jagen-branch (-j) option requires a branch name argument" 103 | jagen_branch="$2"; shift ;; 104 | --root) 105 | init_root=1 ;; 106 | --project) 107 | init_project=1 ;; 108 | -L) [ "$2" ] || die "the -L option requires a directory argument" 109 | init_layer_path="${init_layer_path}${NL}${2}"; shift ;; 110 | -*) die "invalid option: $1" ;; 111 | *) init_layers="${init_layers}${NL}${1}" ;; 112 | esac 113 | shift 114 | done 115 | 116 | jagen_flags=${jagen_flags# } 117 | init_layer_path=${init_layer_path#$NL} 118 | init_layers=${init_layers#$NL} 119 | } 120 | 121 | say_what() { 122 | local action 123 | if is_build_root "$jagen_root_dir"; then 124 | action="reinitializing" 125 | else 126 | action="initializing" 127 | fi 128 | if [ "$jagen_project_dir" ]; then 129 | say "$action a project in '$jagen_project_dir' with a build root in '$jagen_root_dir'" 130 | else 131 | say "$action a workspace in '$jagen_root_dir'" 132 | fi 133 | } 134 | 135 | write_env() { 136 | cat >"$env_file" <"$runner_script" <<'EOF' 158 | #!/bin/sh 159 | . "$(dirname "$0")/env.sh" && _jagen "$@" 160 | EOF 161 | chmod +x "$runner_script" 162 | } 163 | 164 | main() { 165 | local layer path item tmp_list 166 | local layer_count=0 layer_path_count=0 167 | 168 | parse_command_line "$@" 169 | 170 | jagen_dir=$(dirname "$0") 171 | if [ "$jagen_dir" = . ]; then 172 | if ! [ -t 0 ]; then # we are being piped to shell 173 | jagen_dir=".jagen" 174 | else # using Jagen repo dir as a workspace is not supported 175 | show_usage; exit 0 176 | fi 177 | fi 178 | 179 | need_cmd ninja 180 | jagen_ninja_version=$(ninja --version) 181 | 182 | if [ -z "$init_root" -a -z "$init_project" ]; then 183 | if [ "$(ls)" ] && ! is_build_root; then 184 | init_project=1 185 | else 186 | init_root=1 187 | fi 188 | fi 189 | 190 | if [ "$init_root" ]; then 191 | jagen_root_dir=$(real_path) 192 | jagen_project_dir= 193 | elif [ "$init_project" ]; then 194 | jagen_project_dir=$(real_path) 195 | jagen_root_dir=".jagen-root" 196 | fi 197 | 198 | say_what 199 | 200 | for layer in $init_layers; do 201 | case $layer in 202 | /*|./*|../*) [ -d "$layer" ] || 203 | die "the specified layer directory '$layer' does not exist" ;; 204 | esac 205 | done 206 | 207 | for item in $init_layer_path; do 208 | [ -d "$item" ] || 209 | die "the specified layer path directory '$item' does not exist" 210 | done 211 | 212 | git_clone "$jagen_url" "$jagen_dir" $jagen_branch 213 | jagen_dir=$(real_path "$jagen_dir") 214 | 215 | if [ "$init_project" ]; then 216 | [ -d "$jagen_root_dir" ] || mkdir -p "$jagen_root_dir" 217 | 218 | cd "$jagen_root_dir" 219 | jagen_root_dir=$(real_path) 220 | 221 | # assume that the root dir is 1 level below the project dir 222 | tmp_list= 223 | for layer in $init_layers; do 224 | case $layer in 225 | ../*) tmp_list="${tmp_list}${NL}../${layer}" ;; 226 | ./*) tmp_list="${tmp_list}${NL}.${layer}" ;; 227 | *) tmp_list="${tmp_list}${NL}${layer}" ;; 228 | esac 229 | done 230 | init_layers=${tmp_list#$NL} 231 | 232 | tmp_list= 233 | for path in $init_layer_path; do 234 | case $path in 235 | /*) tmp_list="${tmp_list}${NL}${path}" ;; 236 | ./*) tmp_list="${tmp_list}${NL}.${path}" ;; 237 | ../*|*) tmp_list="${tmp_list}${NL}../${path}" ;; 238 | esac 239 | done 240 | init_layer_path=${tmp_list#$NL} 241 | fi 242 | 243 | if ! [ -t 1 ]; then # the output is not connected to a terminal 244 | say "running non-interactively" 245 | export GIT_TERMINAL_PROMPT=0 246 | export GIT_SSH_COMMAND="ssh -o BatchMode=yes" 247 | fi 248 | 249 | for layer in $init_layers; do 250 | layer_count=$((layer_count+1)) 251 | if path_is_url "$layer"; then 252 | path=${layer##*/} path=${path%.git} path="./layer/$path" 253 | git_clone "$layer" "$path" $jagen_layer_branch 254 | layer="$path" 255 | fi 256 | jagen_layers=${jagen_layers}${NL}${layer} 257 | done 258 | 259 | if [ $layer_count -le 1 ]; then 260 | jagen_layers=${jagen_layers#$NL} 261 | jagen_layers=${jagen_layers%$NL} 262 | else 263 | jagen_layers=${jagen_layers}${NL} 264 | fi 265 | 266 | for path in $init_layer_path; do 267 | layer_path_count=$((layer_path_count+1)) 268 | jagen_layer_path=${jagen_layer_path}${NL}${path} 269 | done 270 | 271 | if [ $layer_path_count -le 1 ]; then 272 | jagen_layer_path=${jagen_layer_path#$NL} 273 | jagen_layer_path=${jagen_layer_path%$NL} 274 | else 275 | jagen_layer_path=${jagen_layer_path}${NL} 276 | fi 277 | 278 | write_env 279 | write_runner 280 | 281 | . ./env.sh 282 | jagen refresh 283 | } 284 | 285 | main "$@" 286 | -------------------------------------------------------------------------------- /lib/pkg/android-ndk.lua: -------------------------------------------------------------------------------- 1 | return { 2 | source = { 3 | type = 'dist', 4 | location = 'https://dl.google.com/android/repository/android-ndk-r16b-linux-x86_64.zip', 5 | sha1sum = '42aa43aae89a50d1c66c3f9fdecd676936da6128', 6 | basename = 'android-ndk-r16b' 7 | }, 8 | export = { 9 | env = { 10 | ANDROID_NDK_HOME = "$pkg_source_dir" 11 | }, 12 | cmake_options = { 13 | '-DCMAKE_TOOLCHAIN_FILE=${pkg_source_dir}/build/cmake/android.toolchain.cmake', 14 | } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /lib/pkg/android-ndk~r17b.lua: -------------------------------------------------------------------------------- 1 | return { 2 | source = { 3 | type = 'dist', 4 | location = 'https://dl.google.com/android/repository/android-ndk-r17b-linux-x86_64.zip', 5 | sha1sum = 'dd5762ee7ef4995ad04fe0c45a608c344d99ca9f', 6 | basename = 'android-ndk-r17b' 7 | }, 8 | export = { 9 | env = { 10 | ANDROID_NDK_HOME = "$pkg_source_dir" 11 | } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /lib/pkg/android-sdk-tools.lua: -------------------------------------------------------------------------------- 1 | return { 2 | source = { 3 | location = 'https://dl.google.com/android/repository/sdk-tools-linux-3859397.zip', 4 | basename = 'tools', 5 | sha1sum = '7eab0ada7ff28487e1b340cc3d866e70bcb4286e', 6 | }, 7 | install = true, 8 | export = { 9 | env = { 10 | -- ANROID_HOME is deprecated but Gradle plugin does not know this 11 | ANDROID_HOME = "$pkg_source_dir/..", 12 | ANDROID_SDK_ROOT = "$pkg_export_env_ANDROID_HOME", 13 | } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /lib/pkg/android-sdk-tools.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | jagen_stage_install() { 4 | local sdkmanager="$pkg_source_dir/bin/sdkmanager" 5 | yes | "$sdkmanager" --licenses 6 | message "installing/updating tools using SDK Manager" 7 | pkg_run "$sdkmanager" "cmake;3.6.4111459" 8 | } 9 | -------------------------------------------------------------------------------- /lib/pkg/android-sdk-tools~4333796.lua: -------------------------------------------------------------------------------- 1 | return { 2 | source = { 3 | location = 'https://dl.google.com/android/repository/sdk-tools-linux-4333796.zip', 4 | basename = 'tools', 5 | sha256sum = '92ffee5a1d98d856634e8b71132e8a95d96c83a63fde1099be3d86df3106def9', 6 | }, 7 | install = true, 8 | export = { 9 | env = { 10 | -- ANROID_HOME is deprecated but Gradle plugin does not know this 11 | ANDROID_HOME = "$pkg_source_dir/..", 12 | ANDROID_SDK_ROOT = "$pkg_export_env_ANDROID_HOME", 13 | } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /lib/pkg/android-standalone-arm.lua: -------------------------------------------------------------------------------- 1 | return { 2 | build = { 3 | type = 'android-standalone-toolchain', 4 | toolchain = 'android-ndk', 5 | system = 'arm-linux-androideabi', 6 | cc = 'clang', 7 | cxx = 'clang++', 8 | }, 9 | export = { 10 | -- -- Android 5.0 (API >= 21) only supports PIE 11 | ldflags = '-pie -fPIE', 12 | cmake_options = { 13 | '-DCMAKE_POSITION_INDEPENDENT_CODE=YES', 14 | -- CMake scripts use this to switch behaviour 15 | '-DANDROID=YES', 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /lib/pkg/gdb.lua: -------------------------------------------------------------------------------- 1 | return { 2 | source = { 3 | location = 'https://ftp.gnu.org/gnu/gdb/gdb-7.12.1.tar.xz', 4 | sha256sum = '4607680b973d3ec92c30ad029f1b7dbde3876869e6b3a117d8a7e90081113186' 5 | }, 6 | build = { 7 | type = 'gnu', 8 | options = { 9 | -- specify both build and host explicitly here so a user can set 10 | -- their desired target in a project 11 | '--build=$(jagen_get_system)', 12 | '--host=$(jagen_get_system)', 13 | '--disable-binutils', 14 | '--disable-etc', 15 | '--disable-gas', 16 | '--disable-gold', 17 | '--disable-ld', 18 | '--disable-gprof', 19 | '--disable-gdbserver' 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /lib/pkg/gdb.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | jagen_stage_configure() { 4 | # Workaround from: https://sourceware.org/bugzilla/show_bug.cgi?id=18113#c1 5 | cat > makeinfo <<'EOF' || return 6 | #!/bin/sh 7 | echo "makeinfo (GNU texinfo) 5.2" 8 | EOF 9 | pkg_run chmod a+x makeinfo 10 | pkg_configure MAKEINFO="$(real_path "$PWD")/makeinfo" 11 | } 12 | -------------------------------------------------------------------------------- /lib/pkg/gdbserver.lua: -------------------------------------------------------------------------------- 1 | return { 2 | source = { 3 | location = 'https://ftp.gnu.org/gnu/gdb/gdb-7.12.1.tar.xz', 4 | sha256sum = '4607680b973d3ec92c30ad029f1b7dbde3876869e6b3a117d8a7e90081113186', 5 | dir = 'gdb/gdbserver' 6 | }, 7 | build = 'gnu' 8 | } 9 | -------------------------------------------------------------------------------- /lib/pkg/gdbserver~8.lua: -------------------------------------------------------------------------------- 1 | return { 2 | source = { 3 | location = 'https://ftp.gnu.org/gnu/gdb/gdb-8.2.tar.xz', 4 | sha256sum = 'c3a441a29c7c89720b734e5a9c6289c0a06be7e0c76ef538f7bbcef389347c39', 5 | dir = 'gdb/gdbserver' 6 | }, 7 | build = 'gnu' 8 | } 9 | -------------------------------------------------------------------------------- /lib/pkg/gdb~8.lua: -------------------------------------------------------------------------------- 1 | return { 2 | source = { 3 | location = 'https://ftp.gnu.org/gnu/gdb/gdb-8.2.tar.xz', 4 | sha256sum = 'c3a441a29c7c89720b734e5a9c6289c0a06be7e0c76ef538f7bbcef389347c39' 5 | }, 6 | build = { 7 | type = 'gnu', 8 | options = { 9 | -- specify both build and host explicitly here so a user can set 10 | -- their desired target in a project 11 | '--build=$(jagen_get_system)', 12 | '--host=$(jagen_get_system)', 13 | '--disable-binutils', 14 | '--disable-etc', 15 | '--disable-gas', 16 | '--disable-gold', 17 | '--disable-ld', 18 | '--disable-gprof', 19 | '--disable-gdbserver' 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /lib/pkg/linaro-aarch64-5.3.lua: -------------------------------------------------------------------------------- 1 | return { 2 | source = { 3 | type = 'dist', 4 | location = 'http://releases.linaro.org/components/toolchain/binaries/5.3-2016.05/aarch64-linux-gnu/gcc-linaro-5.3.1-2016.05-x86_64_aarch64-linux-gnu.tar.xz', 5 | sha256sum = '1941dcf6229d6706bcb89b7976d5d43d170efdd17c27d5fe1738e7ecf22adc37', 6 | }, 7 | build = { 8 | system = 'aarch64-linux-gnu', 9 | toolchain = false 10 | }, 11 | install = 'toolchain' 12 | } 13 | -------------------------------------------------------------------------------- /lib/pkg/linaro-arm-4.8.lua: -------------------------------------------------------------------------------- 1 | return { 2 | source = { 3 | type = 'dist', 4 | location = 'https://releases.linaro.org/archive/15.06/components/toolchain/binaries/4.8/arm-linux-gnueabi/gcc-linaro-4.8-2015.06-x86_64_arm-linux-gnueabi.tar.xz', 5 | sha256sum = '04556b1a453008222ab55e4ab9d792f32b6f8566e46e99384225a6d613851587', 6 | }, 7 | build = { 8 | system = 'arm-linux-gnueabi', 9 | toolchain = false 10 | }, 11 | install = 'toolchain' 12 | } 13 | -------------------------------------------------------------------------------- /lib/pkg/linaro-arm-4.9.lua: -------------------------------------------------------------------------------- 1 | return { 2 | source = { 3 | type = 'dist', 4 | location = 'https://releases.linaro.org/components/toolchain/binaries/4.9-2017.01/arm-linux-gnueabi/gcc-linaro-4.9.4-2017.01-x86_64_arm-linux-gnueabi.tar.xz', 5 | sha256sum = '232302236c90ec136a42d44bc387d2e1199f5df010b77e97ee590216d2b5b181', 6 | }, 7 | build = { 8 | system = 'arm-linux-gnueabi', 9 | toolchain = false 10 | }, 11 | install = 'toolchain' 12 | } 13 | -------------------------------------------------------------------------------- /lib/pkg/linaro-arm-5.4.lua: -------------------------------------------------------------------------------- 1 | return { 2 | source = { 3 | type = 'dist', 4 | location = 'https://releases.linaro.org/components/toolchain/binaries/5.4-2017.05/arm-linux-gnueabi/gcc-linaro-5.4.1-2017.05-x86_64_arm-linux-gnueabi.tar.xz', 5 | sha256sum = 'ac60d6831d4053b94e02865dbab6b42ca2be3eda97f82bfa7d6747f4546dcb1f', 6 | }, 7 | build = { 8 | system = 'arm-linux-gnueabi', 9 | toolchain = false 10 | }, 11 | install = 'toolchain' 12 | } 13 | -------------------------------------------------------------------------------- /lib/pkg/linaro-arm-6.4.lua: -------------------------------------------------------------------------------- 1 | return { 2 | source = { 3 | type = 'dist', 4 | location = 'https://releases.linaro.org/components/toolchain/binaries/6.4-2017.11/arm-linux-gnueabi/gcc-linaro-6.4.1-2017.11-x86_64_arm-linux-gnueabi.tar.xz', 5 | sha256sum = 'a7af327e8a07b709eb794ae5ecc21940dee1a60779f0756efc9f8e1ec992f5f6', 6 | }, 7 | build = { 8 | system = 'arm-linux-gnueabi', 9 | toolchain = false 10 | }, 11 | install = 'toolchain' 12 | } 13 | -------------------------------------------------------------------------------- /lib/pkg/linaro-arm-7.2.lua: -------------------------------------------------------------------------------- 1 | package { 'linaro-arm-7.2', 2 | source = { 3 | type = 'dist', 4 | location = 'https://releases.linaro.org/components/toolchain/binaries/7.2-2017.11/arm-linux-gnueabi/gcc-linaro-7.2.1-2017.11-x86_64_arm-linux-gnueabi.tar.xz', 5 | sha256sum = '89e9bfc7ffe615f40a72c2492df0488f25fc20404e5f474501c8d55941337f71', 6 | }, 7 | build = { 8 | system = 'arm-linux-gnueabi' 9 | }, 10 | install = 'toolchain', 11 | export = { 12 | system = '${build.system}' 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /lib/pkg/repo.lua: -------------------------------------------------------------------------------- 1 | return { 2 | source = 'https://storage.googleapis.com/git-repo-downloads/repo', 3 | { 'install' } 4 | } 5 | -------------------------------------------------------------------------------- /lib/pkg/repo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | jagen_stage_install() { 4 | local dest="${pkg_install_dir:?}/bin" 5 | set -- ${pkg_source:?} 6 | 7 | pkg_run mkdir -p "$dest" 8 | if [ -f "${jagen_dist_dir:?}/repo" ]; then 9 | pkg_run cp -v "$jagen_dist_dir/repo" "$dest" 10 | else 11 | pkg__curl "$2" > "$dest/repo" || die 12 | fi 13 | pkg_run chmod +x "$dest/repo" 14 | } 15 | -------------------------------------------------------------------------------- /lib/pkg/rustup.lua: -------------------------------------------------------------------------------- 1 | return { 2 | -- the executable should be named 'rustup-init' or it will not run 3 | -- complaining that the default toolchain is not set 4 | source = { 5 | type = 'dist', 6 | location = 'https://static.rust-lang.org/rustup/dist/$(jagen_get_system)/rustup-init', 7 | }, 8 | build = { 9 | type = 'executable', 10 | system = '$(jagen_get_system)', 11 | options = { 12 | '-y', 13 | '--no-modify-path', 14 | '--default-host', '$pkg_build_system', 15 | '--default-toolchain', 'none' 16 | }, 17 | toolchain = false 18 | }, 19 | install = true, 20 | env = { 21 | RUSTUP_HOME = '$jagen_dist_dir/rustup', 22 | CARGO_HOME = '$jagen_dist_dir/cargo' 23 | }, 24 | export = { 25 | env = { 26 | RUSTUP_HOME = '$pkg_env_RUSTUP_HOME', 27 | CARGO_HOME = '$pkg_env_CARGO_HOME' 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /lib/pkg/sourcery-mips-2012.03.lua: -------------------------------------------------------------------------------- 1 | return { 2 | source = { 3 | type = 'dist', 4 | -- https://sourcery.mentor.com/GNUToolchain/subscription3130?lite=MIPS 5 | location = 'https://sourcery.mentor.com/GNUToolchain/package10395/public/mips-linux-gnu/mips-2012.03-63-mips-linux-gnu-i686-pc-linux-gnu.tar.bz2', 6 | sha256sum = '0a2d92bbca2926479662affcc4dcb644ddca7d8d50ec17ed3b99d7025a552530', 7 | md5sum = '44970134eea6f6ab754a7bbe6e2805d6', 8 | basename = 'mips-2012.03' 9 | }, 10 | build = { 11 | system = 'mips-linux-gnu', 12 | toolchain = false 13 | }, 14 | install = 'toolchain' 15 | } 16 | -------------------------------------------------------------------------------- /lib/pkg/spawn.lua: -------------------------------------------------------------------------------- 1 | return { 2 | source = { 3 | location = 'https://github.com/bazurbat/spawn.git', 4 | rev = 'ec0c7644a04426925cc400076befda7abaeeb926' 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /lib/pkg/system-native-clang.lua: -------------------------------------------------------------------------------- 1 | package { 2 | name = 'system-native-clang', 3 | class = 'toolchain', 4 | source = { 5 | dir = '/usr' 6 | }, 7 | install = 'toolchain', 8 | export = { 9 | cc = 'clang', 10 | cxx = 'clang++', 11 | cflags = { '-march=native' } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /lib/pkg/system-native-gcc.lua: -------------------------------------------------------------------------------- 1 | package { 2 | name = 'system-native-gcc', 3 | class = 'toolchain', 4 | source = { 5 | dir = '/usr' 6 | }, 7 | install = 'toolchain', 8 | export = { 9 | cc = 'gcc', 10 | cxx = 'g++', 11 | ld = 'ld', 12 | cflags = { '-march=native' } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /misc/bash_completion: -------------------------------------------------------------------------------- 1 | ### Jagen Bash completion definitions ### 2 | 3 | _jagen_complete() { 4 | local word="$2" prev_word="$3" 5 | local cur prev bin dir build_dir 6 | local jagen_command jagen_subcommand 7 | 8 | bin=${COMP_WORDS[0]} 9 | dir=${bin%/*} 10 | if [[ "$dir" == "$bin" ]]; then 11 | dir="$jagen_dir" 12 | build_dir="$jagen_build_dir" 13 | else 14 | build_dir=$(source "$dir/env.sh" 15 | echo "$jagen_build_dir") 16 | fi 17 | [ "$dir" ] || return 0 18 | 19 | jagen_command=${COMP_WORDS[1]} 20 | jagen_subcommand=${COMP_WORDS[2]} 21 | 22 | local names_file="$build_dir/.jagen-names" 23 | local scm_names_file="$build_dir/.jagen-scm-names" 24 | local configs_file="$build_dir/.jagen-configs" 25 | local targets_file="$build_dir/.jagen-targets" 26 | local layers_file="$build_dir/.jagen-layers" 27 | 28 | local help_opt='-h --help' 29 | local ignore_opt='-y --ignore-dirty -Y --ignore_exclude' 30 | local top_cmds='help clean refresh build rebuild do re source src list update' 31 | local clean_opt="-m --match -x --exclude $ignore_opt" 32 | local src_cmds='dirty status clean update delete each' 33 | local src_clean_opt="$ignore_opt" 34 | local src_update_opt="$ignore_opt" 35 | local src_each_opt='--type' 36 | local src_types='git hg repo' 37 | local build_opt="-m --match -c --clean -a --all -n --no-rebuild -p --progress -f --follow -F --follow-all -q --quiet -x --exclude $ignore_opt" 38 | local list_packages_opt='--depth -d --all -a' 39 | 40 | _get_comp_words_by_ref -n : cur prev 41 | 42 | if [[ "$prev" =~ jagen$ && ${#COMP_WORDS[*]} -le 2 ]]; then 43 | COMPREPLY=($(compgen -W "$help_opt $top_cmds" -- "$cur")) 44 | return 45 | fi 46 | 47 | case $jagen_command in 48 | clean) 49 | if [[ "$prev" == "clean" ]]; then 50 | COMPREPLY=($(compgen -W "$help_opt $clean_opt $(cat "$configs_file")" -- "$cur")) 51 | else 52 | COMPREPLY=($(compgen -W "$clean_opt $(cat "$configs_file")" -- "$cur")) 53 | fi 54 | ;; 55 | refresh) 56 | COMPREPLY=($(compgen -W "$help_opt" -- "$cur")) ;; 57 | build|do|rebuild|re) 58 | COMPREPLY=($(compgen -W "$help_opt $build_opt $(cat "$names_file") $(cat "$targets_file")" -- "$cur")) ;; 59 | src|source) 60 | if [[ "$prev" == "source" ]] || [[ "$prev" == "src" ]]; then 61 | COMPREPLY=($(compgen -W "$help_opt $src_cmds" -- "$cur")) 62 | else 63 | case $jagen_subcommand in 64 | clean) 65 | COMPREPLY=($(compgen -W "$src_clean_opt $(cat "$scm_names_file")" -- "$cur")) ;; 66 | update) 67 | COMPREPLY=($(compgen -W "$src_update_opt $(cat "$scm_names_file")" -- "$cur")) ;; 68 | each) 69 | if [[ "$prev" == "--type" ]]; then 70 | COMPREPLY=($(compgen -W "$src_types" -- "$cur")) 71 | else 72 | COMPREPLY=($(compgen -W "$help_opt $src_each_opt" -- "$cur")) 73 | fi ;; 74 | *) 75 | COMPREPLY=($(compgen -W "$(cat "$scm_names_file")" -- "$cur")) ;; 76 | esac 77 | fi ;; 78 | list) 79 | if [[ "$prev" == "list" ]]; then 80 | COMPREPLY=($(compgen -W "$help_opt packages layers" -- "$cur")) 81 | elif [[ "$jagen_subcommand" == "packages" ]]; then 82 | COMPREPLY=($(compgen -W "$help_opt $list_packages_opt" -- "$cur")) 83 | fi ;; 84 | update) 85 | COMPREPLY=($(compgen -W "$help_opt jagen self $(cat "$layers_file")" -- "$cur")) ;; 86 | esac 87 | 88 | __ltrim_colon_completions "$cur" 89 | } 90 | 91 | complete -F _jagen_complete jagen 92 | 93 | ### End of Jagen Bash completion definitions ### 94 | -------------------------------------------------------------------------------- /src/Chunk.lua: -------------------------------------------------------------------------------- 1 | local Chunk = {} 2 | 3 | function Chunk:new(obj) 4 | local obj = obj or {} 5 | self._string_ = tostring(obj) 6 | setmetatable(obj, self) 7 | self.__index = self 8 | return obj 9 | end 10 | 11 | function Chunk:each() 12 | return function (_, prev) 13 | local key, value = prev, nil 14 | repeat 15 | key, value = next(self, key) 16 | until key == nil 17 | or type(key) == 'string' 18 | and key:sub(1, 1) ~= '_' 19 | return key, value 20 | end 21 | end 22 | 23 | function Chunk:__tostring() 24 | return self._name_ or self._string_ 25 | end 26 | 27 | return Chunk 28 | -------------------------------------------------------------------------------- /src/Command.lua: -------------------------------------------------------------------------------- 1 | local Log = require 'Log' 2 | 3 | local Command = {} 4 | 5 | function Command:new(...) 6 | local o = { command = {...} } 7 | setmetatable(o, self) 8 | self.__index = self 9 | return o 10 | end 11 | 12 | function Command:newf(f, ...) 13 | return Command:new(string.format(f, ...)) 14 | end 15 | 16 | function Command:append(...) 17 | append(self.command, ...) 18 | return self 19 | end 20 | 21 | function Command:__tostring() 22 | return table.concat(self.command, ' ') 23 | end 24 | 25 | function Command:exists() 26 | return Command:new('command -v', self.command[1]):read() ~= nil 27 | end 28 | 29 | function Command:exec() 30 | local cmdstr = tostring(self) Log.debug1(cmdstr:escape_format()) 31 | local ok, how, status = os.execute(assert(cmdstr)) 32 | if type(ok) == 'number' then 33 | return ok == 0 -- Lua 5.1 34 | else 35 | return status == 0 -- Lua 5.2 36 | end 37 | end 38 | 39 | function Command:popen(mode) 40 | local command = tostring(self) Log.debug1(command:escape_format()) 41 | return assert(io.popen(command, mode)) 42 | end 43 | 44 | function Command:lines() 45 | local pipe = self:popen() 46 | local lines = pipe:lines() 47 | return function () 48 | local line = lines() 49 | if line ~= nil then 50 | return line 51 | else 52 | pipe:close() 53 | end 54 | end 55 | end 56 | 57 | function Command:match(pattern) 58 | for line in self:lines() do 59 | local m = line:match(pattern) 60 | if m then return m end 61 | end 62 | end 63 | 64 | function Command:aslist() 65 | local pipe = self:popen() 66 | local list = aslist(pipe:lines()) 67 | pipe:close() 68 | return list 69 | end 70 | 71 | function Command:read(...) 72 | local file = self:popen() 73 | local results = { file:read(...) } 74 | file:close() 75 | return table.unpack(results) 76 | end 77 | 78 | return Command 79 | -------------------------------------------------------------------------------- /src/Config.lua: -------------------------------------------------------------------------------- 1 | local Rule = require 'Rule' 2 | 3 | local Config = Rule:new() 4 | 5 | function Config:new(rule) 6 | setmetatable(rule, self) 7 | self.__index = self 8 | return rule:_parse(rule) 9 | end 10 | 11 | function Config:_parse(rule) 12 | if type(rule[1]) == 'string' then 13 | self.name = rule[1] 14 | rule[1] = nil 15 | end 16 | return rule 17 | end 18 | 19 | return Config 20 | -------------------------------------------------------------------------------- /src/Context.lua: -------------------------------------------------------------------------------- 1 | 2 | local Context = {} 3 | 4 | function Context:new(o) 5 | o = o or {} 6 | setmetatable(o, self) 7 | self.__index = self 8 | return o 9 | end 10 | 11 | function Context:__rawtostring() 12 | local saved = Context.__tostring 13 | Context.__tostring = nil 14 | local s = tostring(self) 15 | Context.__tostring = saved 16 | return s 17 | end 18 | 19 | function Context:__tostring() 20 | local insert, concat = table.insert, table.concat 21 | local lines = {} 22 | local function append(...) 23 | for i = 1, select('#', ...) do 24 | insert(lines, (select(i, ...))) 25 | end 26 | end 27 | if self.name then 28 | append(self.name) 29 | end 30 | if self.config then 31 | append(':', self.config) 32 | end 33 | if self.filename then 34 | if #lines > 0 then append(' ') end 35 | if self.name or self.config then append('(') end 36 | local filename, removed = self.filename:remove_prefix(System.dirname(Jagen.root_dir)) 37 | if removed then append('...') end append(filename) 38 | if self.line then append(':', self.line) end 39 | if self.name or self.config then append(')') end 40 | end 41 | if self.template then 42 | append(' [', concat(self.template, ', '), ']') 43 | end 44 | if self.implicit and #lines > 0 then 45 | append(' *') 46 | end 47 | return table.concat(lines) 48 | end 49 | 50 | function Context:__unm() 51 | local this = self 52 | local s = '' 53 | while this do 54 | for k, v in pairs(this) do 55 | if k ~= 'parent' then 56 | s = string.format('%s:%s=%s', s, k, tostring(v)) 57 | end 58 | end 59 | this = this.parent 60 | end 61 | return s 62 | end 63 | 64 | function Context:tokey(with_parent) 65 | local o = {} 66 | if with_parent then 67 | if self.name then 68 | table.insert(o, self.name) 69 | end 70 | if self.filename then 71 | table.insert(o, self.filename) 72 | end 73 | if self.line then 74 | table.insert(o, self.line) 75 | end 76 | end 77 | if self.config then 78 | table.insert(o, self.config) 79 | end 80 | if self.template then 81 | table.iextend(o, self.template) 82 | end 83 | return table.concat(o, ':') 84 | end 85 | 86 | local current_context 87 | local context_stack = {} 88 | 89 | 90 | local function push_context(new) 91 | new = Context:new(new) 92 | new.parent = current_context 93 | table.insert(context_stack, new) 94 | current_context = new 95 | return new 96 | end 97 | 98 | local function pop_context() 99 | local o = assert(table.remove(context_stack)) 100 | current_context = context_stack[#context_stack] 101 | return o 102 | end 103 | 104 | 105 | return Context 106 | -------------------------------------------------------------------------------- /src/Engine.lua: -------------------------------------------------------------------------------- 1 | local Command = require 'Command' 2 | local Log = require 'Log' 3 | local Module = require 'Module' 4 | local Package = require 'Package' 5 | local Rule = require 'Rule' 6 | local System = require 'System' 7 | local Target = require 'Target' 8 | 9 | local format = string.format 10 | 11 | local Engine = {} 12 | local ProcessingEnv = {} 13 | 14 | function Engine:new() 15 | local engine = { 16 | path = {}, 17 | modules = {}, 18 | packages = {}, 19 | templates = {}, 20 | } 21 | setmetatable(engine, self) 22 | self.__index = self 23 | return engine 24 | end 25 | 26 | function ProcessingEnv:new(engine) 27 | return { 28 | engine = engine, 29 | packages = {}, 30 | templates = {}, 31 | } 32 | end 33 | 34 | function Engine:error(...) 35 | Log.error(...) 36 | end 37 | 38 | function Engine:load_rules() 39 | Log.debug1('load rules') 40 | 41 | local jagen_rules = System.mkpath(os.getenv('jagen_dir'), 'lib', 'rules.lua') 42 | local root_rules = System.mkpath(os.getenv('jagen_root_dir'), 'rules.lua') 43 | 44 | local jagen = Module:load('jagen', jagen_rules) 45 | append(self.path, jagen:basename(jagen.filename)) 46 | self:process_module(jagen) 47 | 48 | local cmd = Command:new('cmake', '--version') 49 | if cmd:exists() then 50 | local cmake = self.packages.jagen.cmake 51 | cmake.version = cmd:match('^cmake version ([%w_.]+)$') 52 | if Jagen.command._compare_versions{'ge', cmake.version, '3.1'} then 53 | cmake.supports_disable_package_registry = true 54 | end 55 | if Jagen.command._compare_versions{'ge', cmake.version, '3.5'} then 56 | cmake.supports_export_compile_commands = true 57 | end 58 | end 59 | 60 | if System.file_exists(root_rules) then 61 | local root = Module:load('root', root_rules) 62 | append(self.path, root:basename(root.filename)) 63 | self:process_module(root) 64 | end 65 | 66 | local Source = require 'Source' 67 | for pkg in each(self.packages) do 68 | if pkg.source ~= nil then 69 | pkg.source = Source:create(pkg.source, pkg.name) 70 | end 71 | end 72 | 73 | if os.getenv('jagen_debug_engine') then 74 | for pkg in each(self.packages) do 75 | print(pretty(pkg)) 76 | end 77 | end 78 | 79 | self:validate() 80 | 81 | return self.packages 82 | end 83 | 84 | function Engine:finalize() 85 | local path = {} 86 | for dir in each(self.path) do 87 | append(path, string.format('%s/pkg/?.sh', dir)) 88 | end 89 | local script_path = table.concat(path, ';') 90 | 91 | for pkg in each(self.packages) do 92 | pkg._targets = {} 93 | for name, stage in pairs(pkg.stage or {}) do 94 | local target = Target.from_args(pkg.name, name) 95 | -- target.log = System.mkpath(self.packages.jagen.log_dir, target.ref..'.log') 96 | target.inputs = stage.inputs 97 | pkg._targets[name] = target 98 | end 99 | 100 | local filename, err = package.searchpath(pkg.name, script_path, '') 101 | if filename then 102 | pkg.backing_script = filename 103 | end 104 | end 105 | end 106 | 107 | function Engine:process_module(module, pkg) 108 | local env = ProcessingEnv:new(self) 109 | if pkg then 110 | self:add_package(pkg, env) 111 | end 112 | local modules = module:collect_unprocessed(self.modules) 113 | for mod in each(modules) do 114 | Log.debug1('process module %s', mod) 115 | for rule in each(mod.packages) do 116 | self:process_package(rule, env) 117 | end 118 | extend(env.templates, mod.templates) 119 | self.modules[mod.filename] = mod 120 | end 121 | for pkg in each(env.packages) do 122 | for spec in each(pkg.uses) do 123 | self:process_use(spec, pkg) 124 | end 125 | end 126 | for i = 1, #env.packages do 127 | local pkg = env.packages[i] 128 | for ref in each(pkg.extends) do 129 | local parent = self.packages[ref] 130 | if parent then 131 | local this = copy(parent) 132 | this.abstract = nil 133 | this.config = nil 134 | this:merge(pkg, { env = pkg }) 135 | env.packages[i] = this 136 | env.packages[this.ref] = this 137 | else 138 | error(format('package %s extends %s which is not defined', 139 | pkg.ref, ref)) 140 | end 141 | end 142 | end 143 | self:apply_templates(env) 144 | end 145 | 146 | function Engine:process_package(rule, env) 147 | local pkg = self.packages[rule.ref] 148 | if pkg then 149 | Log.debug1('process package %s: merge with an existing instance', rule.ref) 150 | pkg:merge(rule, { env = pkg }) 151 | else 152 | Log.debug1('process package %s: add new instance', rule.ref) 153 | pkg = Package:new(rule.name, rule.config) 154 | pkg:merge(rule, { env = rule }) 155 | self:add_package(pkg, env) 156 | local module = Module:load_package(rule, self.path) 157 | if module then 158 | self:process_module(module) 159 | end 160 | end 161 | end 162 | 163 | function Engine:process_use(spec, pkg) 164 | local target = Target.from_use(spec) 165 | local use = self.packages[target.ref] 166 | if use then 167 | Log.debug1('process use %s of %s: already processed', target.ref, pkg.ref) 168 | else 169 | Log.debug1('process use %s of %s: add new instance', target.ref, pkg.ref) 170 | use = Package:new(target.name, target.config) 171 | local module = Module:load_package(target, self.path) 172 | if module then 173 | self:process_module(module, use) 174 | else 175 | error(format("the package '%s' uses '%s' which is not defined".. 176 | "and not found in module search path", pkg.ref, target.ref)) 177 | end 178 | end 179 | end 180 | 181 | function Engine:add_package(pkg, pass) 182 | self.packages[pkg.ref] = pkg 183 | append(pass.packages, pkg) 184 | end 185 | 186 | function Engine:apply_templates(pass) 187 | Log.debug1('%d new templates', #pass.templates) 188 | if next(pass.templates) then 189 | for pkg in each(self.packages) do 190 | if not pkg.abstract then 191 | Log.debug1('apply new templates to %s', pkg) 192 | for template in each(pass.templates) do 193 | self:apply_template(template, pkg) 194 | end 195 | end 196 | end 197 | extend(self.templates, pass.templates) 198 | end 199 | Log.debug1('%d new packages', #pass.packages) 200 | if next(pass.packages) then 201 | for pkg in each(pass.packages) do 202 | if not pkg.abstract then 203 | Log.debug1('apply templates to %s', pkg) 204 | for template in each(self.templates) do 205 | self:apply_template(template, pkg) 206 | end 207 | Log.debug1('end apply templates to %s', pkg) 208 | end 209 | end 210 | extend(self.packages, pass.packages) 211 | end 212 | end 213 | 214 | function Engine:apply_template(template, pkg) 215 | local state = { 216 | debug = template.debug, 217 | packages = self.packages, 218 | matching = true, 219 | value = {}, 220 | env = pkg, 221 | self = pkg, 222 | i = 0, n = 1 223 | } 224 | if state.debug then 225 | Log.debug1('apply template %s', pretty(template)) 226 | end 227 | if pkg:match(template.match, state) then 228 | state.matching = false 229 | for i = 1, state.n do 230 | state.i = i 231 | pkg:merge(template.apply, state) 232 | for spec in each(template.apply.uses) do 233 | if type(spec) == 'function' then 234 | spec = spec(state) 235 | end 236 | self:process_use(spec, pkg) 237 | end 238 | end 239 | end 240 | end 241 | 242 | function Engine:validate() 243 | local num_fatal = 0 244 | 245 | local unexpanded, empty = {}, {} 246 | 247 | local function find_invalid(keypath, value) 248 | local tvalue = type(value) 249 | if tvalue == 'string' then 250 | if value:match('${.+}') then 251 | unexpanded[keypath] = value 252 | elseif value:match('') then 253 | empty[keypath] = value 254 | end 255 | elseif tvalue == 'table' then 256 | for k, v in pairs(value) do 257 | find_invalid(string.format('%s.%s', keypath, k), v) 258 | end 259 | end 260 | end 261 | 262 | for pkg in each(self.packages) do 263 | find_invalid(pkg.ref, pkg) 264 | end 265 | 266 | if next(unexpanded) then 267 | local keys = table.keys(unexpanded) 268 | table.sort(keys) 269 | num_fatal = num_fatal + 1 270 | Log.error('Unexpanded properties left after rule loading:') 271 | for key in each(keys) do 272 | Log.error(' %s = %s', key, unexpanded[key]) 273 | end 274 | end 275 | 276 | if next(empty) then 277 | local keys = table.keys(empty) 278 | table.sort(keys) 279 | num_fatal = num_fatal + 1 280 | Log.error('Some properties still contain empty value marker :') 281 | for key in each(keys) do 282 | Log.error(' %s = %s', key, empty[key]) 283 | end 284 | end 285 | 286 | if num_fatal > 0 then 287 | error(string.format('Rule validation failed with %d fatal errors.', num_fatal), 0) 288 | end 289 | end 290 | 291 | return Engine 292 | -------------------------------------------------------------------------------- /src/Log.lua: -------------------------------------------------------------------------------- 1 | local P = {} 2 | 3 | local debug_level = tonumber(os.getenv('jagen_debug')) or -1 4 | 5 | function P.message(...) 6 | io.write('(I) ', string.format(...), '\n') 7 | io.flush() 8 | end 9 | 10 | function P.warning(...) 11 | io.stderr:write('(W) ', string.format(...), '\n') 12 | io.stderr:flush() 13 | end 14 | 15 | function P.error(...) 16 | io.stderr:write('(E) ', string.format(...), '\n') 17 | io.stderr:flush() 18 | end 19 | 20 | function P.debug(...) 21 | if debug_level >= 0 then 22 | io.write('(D0) ', string.format(...), '\n') 23 | io.flush() 24 | end 25 | end 26 | 27 | function P.debug1(...) 28 | if debug_level >= 1 then 29 | io.write('(D1) ', string.format(...), '\n') 30 | io.flush() 31 | end 32 | end 33 | 34 | function P.debug2(...) 35 | if debug_level >= 2 then 36 | io.write('(D2) ', string.format(...), '\n') 37 | io.flush() 38 | end 39 | end 40 | 41 | return P 42 | -------------------------------------------------------------------------------- /src/Ninja.lua: -------------------------------------------------------------------------------- 1 | local P = {} 2 | local System = require 'System' 3 | local Target = require 'Target' 4 | local Command = require 'Command' 5 | 6 | local format = string.format 7 | local concat = table.concat 8 | 9 | local packages = {} 10 | local ninja = { 11 | supports_console_pool = true 12 | } 13 | 14 | local function check_ninja_features() 15 | local version = os.getenv('jagen_ninja_version') 16 | if not version then return end 17 | local major, minor = version:match('(%d+)%.(%d+)%.%d+') 18 | major, minor = tonumber(major), tonumber(minor) 19 | if major and minor then 20 | if major == 1 and minor < 5 then 21 | ninja.supports_console_pool = false 22 | end 23 | end 24 | end 25 | 26 | local function dsp(arg) 27 | end 28 | 29 | local function fmt(args) 30 | for i, arg in ipairs(args) do 31 | if type(arg) == 'function' then 32 | args[i] = arg() 33 | end 34 | end 35 | return table.concat(args) 36 | end 37 | 38 | local function cat(...) 39 | return fmt({...}) 40 | end 41 | 42 | local function comp(...) 43 | local args = {...} 44 | return function(value) 45 | local output = value 46 | for i = 1, #args do 47 | local f = args[i] 48 | if type(f) == 'function' then 49 | output = f(output) 50 | end 51 | end 52 | return output 53 | end 54 | end 55 | 56 | local function findent(n) 57 | return function(value) 58 | return string.format('%s%s', string.rep('^', n), tostring(value)) 59 | end 60 | end 61 | 62 | local function fjoin(args, f, sep) 63 | if f == nil then 64 | f = function(val) return val end 65 | end 66 | return function() 67 | local out = {} 68 | for _, arg in ipairs(args) do 69 | table.insert(out, f(arg)) 70 | end 71 | return table.concat(out, sep) 72 | end 73 | end 74 | 75 | local function with_prefix(prefix) 76 | return function(v) 77 | return string.format('%s%s', prefix, tostring(v)) 78 | end 79 | end 80 | 81 | local function with_suffix(suffix) 82 | return function(v) 83 | return string.format('%s%s', tostring(v), suffix) 84 | end 85 | end 86 | 87 | local function escaped() 88 | return function(s) 89 | s = string.gsub(s, "%$", "$$") 90 | s = string.gsub(s, " ", "$ ") 91 | s = string.gsub(s, ":", "$:") 92 | return s 93 | end 94 | end 95 | 96 | local function indent(n) 97 | return string.rep(' ', n or 4) 98 | end 99 | 100 | local function indented(line, n) 101 | return concat { indent(n), line } 102 | end 103 | 104 | local function separated(str) 105 | if not string.empty(str) then 106 | return str..(suffix or ' ') 107 | else 108 | return '' 109 | end 110 | end 111 | 112 | local function escape(s) 113 | s = string.gsub(s, "%$", "$$") 114 | s = string.gsub(s, " ", "$ ") 115 | s = string.gsub(s, ":", "$:") 116 | return s 117 | end 118 | 119 | local function nonempty(list) 120 | local out = {} 121 | for i = 1, #list do 122 | local item = list[i] 123 | if item and item ~= '' then 124 | table.insert(out, item) 125 | end 126 | end 127 | return out 128 | end 129 | 130 | local function join(list, sep) 131 | return concat(list, sep) 132 | end 133 | 134 | local function join_space(list) 135 | return concat(list, ' ') 136 | end 137 | 138 | local function join_nl(list) 139 | return concat(list, '\n') 140 | end 141 | 142 | local function join_escaped(list) 143 | return concat(list, ' $\n') 144 | end 145 | 146 | local function join_quoted(list) 147 | return join_space(collect(list, map(function (i) 148 | return format("'%s'", escape(tostring(i))) 149 | end))) 150 | end 151 | 152 | local function quote(s) 153 | return format("'%s'", string.gsub(s or '', "%$", "$$")) 154 | end 155 | 156 | local function binding(k, v) 157 | return format('%s = %s', assert(k), tostring(assert(v))) 158 | end 159 | 160 | local function format_pool(name, depth) 161 | return format('pool %s\n%sdepth = %s', name, indent(4), depth) 162 | end 163 | 164 | local function format_rule(name, command) 165 | return format('rule %s\n%scommand = %s', name, indent(4), command) 166 | end 167 | 168 | local function format_target(target) 169 | local output = { target.name, target.stage, target.config } 170 | return join(output, ':') 171 | end 172 | 173 | local function format_outputs(outputs) 174 | local lines = { escape(outputs[1] or '') } 175 | if #outputs > 1 then 176 | extend(lines, map(function (x) 177 | return indented(escape(tostring(x)), 6) 178 | end, sort(table.rest(outputs, 2)))) 179 | append(lines, indent(12)) 180 | end 181 | return join_escaped(lines) 182 | end 183 | 184 | local function format_inputs(inputs) 185 | local lines = { '' } 186 | extend(lines, sort(map(function (x) 187 | return indented(escape(format_target(x)), 16) 188 | end, inputs or {}))) 189 | return join_escaped(lines) 190 | end 191 | 192 | local function format_refresh(files) 193 | local outputs = { 'build.ninja' } 194 | return fmt { '\n', 'build build.ninja: refresh', 195 | fjoin(files, comp(escape, with_prefix(cat(' $\n', indent(6))))), 196 | '\n', indent(4), binding('description', 'refresh') 197 | } 198 | end 199 | 200 | local function format_phony(files) 201 | return format('build %s: phony', format_outputs(files)) 202 | end 203 | 204 | local function format_build(build) 205 | return fmt { '\n', 'build ', 206 | fjoin(build.outputs, comp(escape, with_suffix(' $\n')), indent(6)), 207 | cat(indent(8), ': stage'), 208 | fjoin(build.inputs, comp(escape, with_prefix(cat(' $\n', indent(6))))), 209 | fjoin(map(function (key) 210 | return binding(key, build.vars[key]) 211 | end, sort(table.keys(build.vars))), 212 | with_prefix(cat('\n', indent(2)))) 213 | } 214 | end 215 | 216 | local function format_stage(name, target, pkg) 217 | local output = { pkg.name, name, pkg.config } 218 | local args = { name, quote(pkg.script) } 219 | 220 | local inputs = {} 221 | for _, item in ipairs(target.inputs or {}) do 222 | local t = { item.name or pkg.name, item.stage, item.config or pkg.config } 223 | table.insert(inputs, join(t, ':')) 224 | end 225 | 226 | local vars = { 227 | description = join(output, ' '), 228 | args = join(args, ' '), 229 | } 230 | 231 | vars.pool = target.pool 232 | 233 | return format_build { 234 | rule = 'stage', 235 | uses = {}, 236 | inputs = inputs, 237 | outputs = { join(output, ':') }, 238 | vars = vars 239 | } 240 | end 241 | 242 | local function format_package(pkg) 243 | local lines = {} 244 | local targets = table.keys(pkg._targets) 245 | table.sort(targets) 246 | for name in each(targets) do 247 | append(lines, format_stage(name, pkg._targets[name], pkg)) 248 | end 249 | return join(lines) 250 | end 251 | 252 | local function assign_pools(packages) 253 | local function is_android_gradle(target, pkg) 254 | if target.stage == 'compile' then 255 | local build = pkg:get('build', target.config) 256 | return build and build.type == 'android-gradle' 257 | end 258 | end 259 | local function is_rust_toolchain(target, pkg) 260 | if target.stage == 'install' then 261 | local build = pkg:get('build', target.config) 262 | return build and build.type == 'rust-toolchain' 263 | end 264 | end 265 | local function is_interactive(target) 266 | return target.interactive 267 | end 268 | local function assign_interactive(targets) 269 | if table.find(targets, is_interactive) then 270 | for target in each(targets) do 271 | target.interactive = true 272 | end 273 | end 274 | end 275 | local gradle_stages, rust_stages = {}, {} 276 | for name, pkg in pairs(packages) do 277 | for target, this in pkg:each() do 278 | if is_android_gradle(target, pkg) then 279 | append(gradle_stages, target) 280 | target.pool = 'gradle_android' 281 | end 282 | if is_rust_toolchain(target, pkg) then 283 | append(rust_stages, target) 284 | target.pool = 'rust_toolchain' 285 | end 286 | end 287 | end 288 | assign_interactive(gradle_stages) 289 | assign_interactive(rust_stages) 290 | if ninja.supports_console_pool then 291 | for name, pkg in pairs(packages) do 292 | for target, this in pkg:each() do 293 | if is_interactive(target) then 294 | target.pool = 'console' 295 | end 296 | end 297 | end 298 | end 299 | end 300 | 301 | function P.generate(rules, jagen) 302 | check_ninja_features() 303 | 304 | local out_file = jagen.build_file 305 | 306 | packages = rules 307 | local file = assert(io.open(out_file, 'w')) 308 | 309 | assign_pools(rules) 310 | 311 | local lines = { 312 | binding('ninja_required_version', '1.1'), 313 | binding('builddir', assert(jagen.build_dir)), 314 | format_pool('gradle_android', 1), 315 | format_pool('rust_toolchain', 1), 316 | format_rule('stage', join_space { 317 | -- join { 'jagen_dir=', quote(config.dir) }, 318 | -- join { 'jagen_root_dir=', quote(config.root_dir) }, 319 | join_space { quote(jagen.dir..'/bin/jagen-stage'), '$args' } 320 | }), 321 | format_rule('refresh', join_space(nonempty { jagen.shell, System.expand('$jagen_root_dir/jagen'), 'refresh' })) 322 | } 323 | 324 | local for_refresh = Command:new(jagen.self.cmd, 'find_for_refresh'):aslist() 325 | 326 | append(lines, 'build refresh: phony build.ninja') 327 | append(lines, format_refresh(for_refresh)) 328 | 329 | extend(lines, map(format_package, rules)) 330 | 331 | file:write(join_nl(lines)) 332 | file:write('\n') 333 | 334 | file:close() 335 | end 336 | 337 | return P 338 | -------------------------------------------------------------------------------- /src/Options.lua: -------------------------------------------------------------------------------- 1 | local Log = require 'Log' 2 | 3 | local P = {} 4 | P.__index = P 5 | 6 | function P:new(config) 7 | local this = { 8 | config = { long = {}, short = {} }, 9 | init = {} 10 | } 11 | setmetatable(this, self) 12 | 13 | for _, item in ipairs(config or {}) do 14 | local opt, cfg = {}, '' 15 | if type(item[1]) == 'string' then 16 | cfg = item[1] 17 | table.remove(item, 1) 18 | end 19 | local names, value = table.unpack(cfg:split('=')) 20 | local long, short = table.unpack(names:split(',')) 21 | opt.long = assert(long) 22 | if short then opt.short = short end 23 | if value then 24 | opt.needs_value = value 25 | end 26 | if item[1] ~= nil then 27 | this.init[long] = item[1] 28 | table.remove(item, 1) 29 | else 30 | this.init[long] = false 31 | end 32 | if item[1] ~= nil then 33 | opt.max_value = item[1] 34 | table.remove(item, 1) 35 | end 36 | this.config.long[long] = opt 37 | if short and #short > 0 then 38 | this.config.short[short] = opt 39 | end 40 | end 41 | 42 | return this 43 | end 44 | 45 | function P:is_option(arg) 46 | return string.sub(arg, 1, 1) == '-' 47 | end 48 | 49 | function P:is_short(arg) 50 | return arg and string.sub(arg, 1, 1) == '-' and string.sub(arg, 2, 2) ~= '-' 51 | end 52 | 53 | function P:is_long(arg) 54 | return arg and string.sub(arg, 1, 1) == '-' and string.sub(arg, 2, 2) == '-' 55 | end 56 | 57 | function P:is_eoa(arg) 58 | return P:is_long(arg) and #arg == 2 59 | end 60 | 61 | local function result_tostring(self) 62 | local lines = {} 63 | local function append(value) 64 | table.insert(lines, value) 65 | end 66 | for k, v in pairs(self) do 67 | if v == true or (type(v) == 'table' and 68 | (v.short == true or v.long == true)) 69 | then 70 | append(string.format('--%s', k)) 71 | end 72 | end 73 | return table.concat(lines, ' ') 74 | 75 | end 76 | 77 | function P:parse(args) 78 | local result = copy(self.init) 79 | setmetatable(result, { __tostring = result_tostring }) 80 | local read_nth, eoa 81 | 82 | local function set_value(n, opt, val) 83 | if opt.needs_value == 'n' then 84 | local num = tonumber(val) 85 | if not num then 86 | Log.error("option '%s' (%s) requires a number value but the specified value '%s' is not a number", 87 | opt.long, opt.short, val) 88 | return 89 | end 90 | result[opt.long] = num 91 | else 92 | result[opt.long] = val 93 | end 94 | return true 95 | end 96 | 97 | local function read_value(n, opt) 98 | local arg = args[n] 99 | local value 100 | if arg and not self:is_option(arg) then 101 | value = arg 102 | else 103 | value = opt.max_value 104 | end 105 | if not set_value(n, opt, value) then 106 | return 107 | end 108 | return read_nth(n+1) 109 | end 110 | 111 | local function read_long(n) 112 | local patterns = { '^%-%-([%w-]+)=(.*)$', '^%-%-([%w-]+)' } 113 | local arg = args[n] 114 | local name, value 115 | for pattern in each(patterns) do 116 | name, value = string.match(arg, pattern) 117 | if name then break end 118 | end 119 | local opt = self.config.long[name] 120 | if not opt then 121 | Log.error('invalid option: %s', arg) 122 | return 123 | end 124 | if opt.needs_value then 125 | if value then 126 | if not set_value(n, opt, value) then return end 127 | else 128 | return read_value(n+1, opt) 129 | end 130 | else 131 | if not result[name] then 132 | result[name] = {} 133 | end 134 | result[name].long = true 135 | end 136 | return read_nth(n+1) 137 | end 138 | 139 | local function read_optstring(n, optstring) 140 | local a = string.sub(optstring, 1, 1) 141 | if #a < 1 then 142 | return read_nth(n+1) 143 | end 144 | local rest = string.sub(optstring, 2) 145 | local opt = self.config.short[a] 146 | if not opt then 147 | Log.error('invalid option: -%s', a) 148 | return 149 | end 150 | if opt.needs_value then 151 | if #rest > 0 then 152 | if set_value(n, opt, rest) then 153 | return read_nth(n+1) 154 | else 155 | return 156 | end 157 | else 158 | return read_value(n+1, opt) 159 | end 160 | else 161 | if not result[opt.long] then 162 | result[opt.long] = {} 163 | end 164 | result[opt.long].short = true 165 | return read_optstring(n, rest) 166 | end 167 | end 168 | 169 | local function read_short(n) 170 | local arg = args[n] 171 | return read_optstring(n, string.sub(arg, 2)) 172 | end 173 | 174 | read_nth = function (n) 175 | local arg = args[n] 176 | if not arg then return result end 177 | if eoa then 178 | result._args = append(result._args, args[n]) 179 | else 180 | if self:is_eoa(arg) then 181 | eoa = true 182 | elseif self:is_long(arg) then 183 | return read_long(n) 184 | elseif self:is_short(arg) then 185 | return read_short(n) 186 | else 187 | table.insert(result, args[n]) 188 | end 189 | end 190 | return read_nth(n+1, args) 191 | end 192 | 193 | return read_nth(1) 194 | end 195 | 196 | return P 197 | -------------------------------------------------------------------------------- /src/Package.lua: -------------------------------------------------------------------------------- 1 | local Rule = require 'Rule' 2 | local Source = require 'Source' 3 | 4 | local Package = Rule:new() 5 | 6 | function Package:new(name, config) 7 | local pkg = { name = assert(name), config = config } 8 | pkg.ref = string.format('%s%s', name, config and ':'..config or '') 9 | setmetatable(pkg, self) 10 | self.__index = self 11 | return pkg 12 | end 13 | 14 | function Package:from_rule(rule) 15 | rule = Package:parse(rule) 16 | setmetatable(rule, self) 17 | self.__index = self 18 | return rule 19 | end 20 | 21 | function Package:__tostring(sep) 22 | local c = {} 23 | if self.name then table.insert(c, self.name) end 24 | if self.config then table.insert(c, self.config) end 25 | return table.concat(c, sep or ':') 26 | end 27 | 28 | function Package:create(name) 29 | local pkg = { 30 | name = name, 31 | } 32 | setmetatable(pkg, self) 33 | return pkg 34 | end 35 | 36 | function Package:parse(rule) 37 | if type(rule[1]) == 'string' then 38 | rule.name = rule[1] 39 | table.remove(rule, 1) 40 | end 41 | if type(rule[2]) == 'string' then 42 | rule.config = rule[1] 43 | table.remove(rule, 1) 44 | end 45 | 46 | if rule.name and rule.config then 47 | rule.ref = string.format('%s:%s', rule.name, rule.config) 48 | else 49 | rule.ref = rule.name or rule.config 50 | end 51 | 52 | rule.source = Source:parse(rule.source) 53 | 54 | for key in each { 'class', 'uses', 'extends' } do 55 | local value = rule[key] 56 | if type(value) == 'string' then 57 | rule[key] = { value } 58 | end 59 | end 60 | 61 | for key in each { 'build', 'install' } do 62 | local value = rule[key] 63 | if type(value) == 'string' then 64 | rule[key] = { type = value } 65 | end 66 | end 67 | 68 | if type(rule.patches) == 'table' then 69 | local patches = rule.patches 70 | for i = 1, #patches do 71 | local item = patches[i] 72 | if type(item) == 'string' then 73 | patches[i] = { item, 1 } 74 | elseif type(item) == 'table' then 75 | if not item[2] then 76 | item[2] = 1 77 | end 78 | end 79 | end 80 | end 81 | 82 | if type(rule.files) == 'string' then 83 | rule.files = { rule.files } 84 | end 85 | if type(rule.files) == 'table' then 86 | local files = rule.files 87 | for i = 1, #files do 88 | local item = files[i] 89 | if type(item) == 'string' then 90 | files[i] = { item } 91 | elseif type(item) == 'table' then 92 | if not item.path and item.dir then 93 | item.path = item.dir..'/'..item[1] 94 | end 95 | end 96 | end 97 | end 98 | 99 | return rule 100 | end 101 | 102 | local function collect_array(array, result) 103 | result = result or {} 104 | for key, value in ipairs(array) do 105 | local tvalue = type(value) 106 | if tvalue == 'string' then 107 | append(result, value) 108 | elseif tvalue == 'number' then 109 | append(result, tostring(value)) 110 | elseif tvalue == 'table' then 111 | collect_array(value, result) 112 | end 113 | end 114 | return result 115 | end 116 | 117 | local function format_table(path, value, fmt) 118 | local tvalue = type(value) 119 | if tvalue == 'string' then 120 | fmt(path, value) 121 | elseif tvalue == 'number' then 122 | fmt(path, tostring(value)) 123 | elseif tvalue == 'boolean' and value then 124 | fmt(path, 'yes') 125 | elseif tvalue == 'table' then 126 | for k, v in kvpairs(value) do 127 | format_table(append(copy(path), k), v, fmt) 128 | end 129 | if #value > 0 then 130 | local array = collect_array(value) 131 | if #array > 0 then 132 | fmt(path, table.concat(array, '\t')) 133 | end 134 | end 135 | end 136 | end 137 | 138 | local function format_rule(property, prefix, skip) 139 | skip = skip or {} 140 | 141 | local format = string.format 142 | local result = {} 143 | 144 | local function format_variable(path, value) 145 | local name = string.to_identifier(table.concat(path, '_')) 146 | if value == '' then 147 | append(result, format('unset %s', name)) 148 | else 149 | append(result, format("export %s='%s'", name, value)) 150 | end 151 | end 152 | 153 | for key, value in pairs(property) do 154 | if not skip[key] and key:sub(1, 1) ~= '_' then 155 | local path = prefix and {prefix, key} or {key} 156 | format_table(path, value, format_variable) 157 | end 158 | end 159 | 160 | table.sort(result) 161 | 162 | return table.concat(result, '\n') 163 | end 164 | 165 | function Package:generate_script() 166 | local output = {} 167 | append(output, format_rule(self, 'pkg', { 168 | env = true, 169 | export = true, 170 | })) 171 | if self.export then 172 | append(output, '') 173 | append(output, format_rule(self.export, 'pkg_export')) 174 | end 175 | return table.concat(output, '\n') 176 | end 177 | 178 | function Package:generate_env_script() 179 | if self.env then 180 | return format_rule(self.env) 181 | end 182 | end 183 | 184 | function Package:generate_export_script() 185 | local output = {} 186 | if self.export then 187 | append(output, format_rule(self.export, self.name, { env = true })) 188 | if self.export.env then 189 | append(output, '') 190 | append(output, format_rule(self.export.env)) 191 | end 192 | end 193 | return table.concat(output, '\n') 194 | end 195 | 196 | return Package 197 | -------------------------------------------------------------------------------- /src/Refresh.lua: -------------------------------------------------------------------------------- 1 | local Command = require 'Command' 2 | local Engine = require 'Engine' 3 | local Ninja = require 'Ninja' 4 | local Script = require 'Script' 5 | local System = require 'System' 6 | local Target = require 'Target' 7 | 8 | local Refresh = {} 9 | 10 | local function generate_cargo_config(packages) 11 | local targets, lines = {}, {} 12 | for name, pkg in pairs(packages) do 13 | for this, config in pkg:each_config() do 14 | local build = this.build 15 | if build.type == 'rust' then 16 | local system = pkg:get_build('system', config) 17 | local cc = pkg:get_build('cc', config) 18 | or pkg:get_toolchain_build('cc', config, packages) 19 | or 'gcc' 20 | local toolchain_system = pkg:get_toolchain_build('system', config, packages) 21 | if system and cc and toolchain_system then 22 | targets[system] = string.format('%s-%s', toolchain_system, cc) 23 | end 24 | end 25 | end 26 | end 27 | for target, path in pairs(targets) do 28 | table.insert(lines, string.format('[target.%s]\nlinker = "%s"', target, path)) 29 | end 30 | local config_dir = assert(os.getenv('jagen_cargo_config_dir')) 31 | local config_path = System.mkpath(config_dir, 'config') 32 | System.mkdir(config_dir) 33 | local file = assert(io.open(config_path, 'w')) 34 | file:write(table.concat(lines, '\n'), '\n') 35 | file:close() 36 | end 37 | 38 | function Refresh:run(args) 39 | local format, mkpath = string.format, System.mkpath 40 | 41 | local engine = Engine:new() 42 | local packages = engine:load_rules() 43 | 44 | engine:finalize() 45 | 46 | local root_config = engine.packages.jagen 47 | 48 | local build_dir = root_config.build_dir 49 | local include_dir = root_config.include_dir 50 | 51 | System.mkdir(build_dir, include_dir) 52 | 53 | for pkg in each(packages) do 54 | local scripts = { 55 | { 56 | name = format('%s.sh', pkg.ref), 57 | contents = pkg:generate_script() 58 | }, 59 | { 60 | name = format('%s.env.sh', pkg.ref), 61 | contents = pkg:generate_env_script() 62 | }, 63 | { 64 | name = format('%s.export.sh', pkg.ref), 65 | contents = pkg:generate_export_script() 66 | } 67 | } 68 | 69 | for _, script in ipairs(scripts) do 70 | if script.contents then 71 | local path = mkpath(include_dir, script.name) 72 | local file = assert(io.open(path, 'w')) 73 | file:write(script.contents) 74 | file:close() 75 | end 76 | end 77 | 78 | pkg.script = mkpath(include_dir, scripts[1].name) 79 | end 80 | 81 | Ninja.generate(packages, root_config) 82 | 83 | local autocomplete = { 84 | names = { name = '.jagen-names' }, 85 | scm_names = { name = '.jagen-scm-names' }, 86 | configs = { name = '.jagen-configs' }, 87 | targets = { name = '.jagen-targets' }, 88 | } 89 | 90 | for pkg in each(packages) do 91 | if not pkg.abstract then 92 | append(autocomplete.names, pkg.name) 93 | if pkg.source and pkg.source.scm then 94 | append(autocomplete.scm_names, pkg.name) 95 | end 96 | for stage in pairs(pkg.stage or {}) do 97 | append(autocomplete.targets, pkg.name..':'..stage) 98 | end 99 | end 100 | end 101 | 102 | for _, item in pairs(autocomplete) do 103 | table.sort(item) 104 | System.write_file(mkpath(build_dir, item.name), table.concat(item, '\n')) 105 | 106 | end 107 | end 108 | 109 | return Refresh 110 | -------------------------------------------------------------------------------- /src/Rule.lua: -------------------------------------------------------------------------------- 1 | local Chunk = require 'Chunk' 2 | local Log = require 'Log' 3 | 4 | local Rule = Chunk:new() 5 | 6 | function Rule:new(def) 7 | def = def or {} 8 | setmetatable(def, self) 9 | self.__index = self 10 | return def 11 | end 12 | 13 | function Rule.match(value, pattern, state) 14 | local debug = state.debug 15 | if type(pattern) == 'function' then 16 | if debug then 17 | Log.debug1('match %s -> %s', pattern, pattern(state, value)) 18 | end 19 | return pattern(state, value) 20 | elseif type(value) ~= type(pattern) then 21 | return false 22 | elseif type(value) == 'table' then 23 | for k, v in pairs(pattern) do 24 | if debug then 25 | Log.debug2('k %s, v %s', k, v) 26 | end 27 | if not Rule.match(value[k], v, state) then 28 | return false 29 | end 30 | end 31 | elseif type(value) == 'string' 32 | and not string.match(value, pattern) then 33 | return false 34 | elseif type(value) ~= 'string' and value ~= pattern then 35 | return false 36 | end 37 | return true 38 | end 39 | 40 | function Rule.merge(to, from, state) 41 | for key, value in pairs(from or {}) do 42 | local tkey, tvalue = type(key), type(value) 43 | if tkey ~= 'number' then 44 | if tkey == 'function' then 45 | key = key(state) 46 | end 47 | if tvalue == 'function' then 48 | value = value(state, to[key]) 49 | end 50 | if tvalue == 'table' then 51 | if type(to[key]) ~= 'table' then 52 | to[key] = {} 53 | end 54 | to[key] = Rule.merge(to[key], value, state) 55 | else 56 | to[key] = value 57 | end 58 | end 59 | end 60 | 61 | for i, value in ipairs(from or {}) do 62 | if type(value) == 'function' then 63 | value = value(state) 64 | end 65 | if value ~= nil then 66 | if type(value) == 'table' then 67 | value = Rule.merge({}, value, state) 68 | end 69 | if value ~= nil then 70 | append(to, value) 71 | end 72 | end 73 | end 74 | 75 | return to 76 | end 77 | 78 | return Rule 79 | -------------------------------------------------------------------------------- /src/Script.lua: -------------------------------------------------------------------------------- 1 | local Chunk = require 'Chunk' 2 | local Module = require 'Module' 3 | local System = require 'System' 4 | local Target = require 'Target' 5 | 6 | local P = {} 7 | 8 | local function is_simple_type(t) 9 | return t == 'string' or t == 'number' or t == 'boolean' 10 | end 11 | 12 | local function is_table_type(t) 13 | return t == 'table' 14 | end 15 | 16 | local function is_simple_value(value) 17 | local t = type(value) 18 | return t == 'string' or t == 'number' or t == 'boolean' 19 | end 20 | 21 | local function write_array(value) 22 | local t = type(value) 23 | if is_simple_type(t) then 24 | return tostring(value) 25 | elseif is_table_type(t) then 26 | local result = {} 27 | for i, v in ipairs(value) do 28 | append(result, write_array(v)) 29 | end 30 | return table.concat(result, '\t') 31 | end 32 | end 33 | 34 | local function write(fmt, name, value) 35 | local tvalue = type(value) 36 | name = string.to_identifier(name) 37 | if tvalue == 'string' then 38 | fmt("%s='%s'", name, value) 39 | elseif tvalue == 'number' then 40 | fmt("%s='%s'", name, tostring(value)) 41 | elseif tvalue == 'boolean' and value then 42 | fmt("%s='true'", name) 43 | elseif tvalue == 'table' then 44 | local keys = table.keys(value) 45 | table.sort(keys) 46 | for key in each(keys) do 47 | local newname = key 48 | if #name > 0 then 49 | newname = string.format('%s_%s', name, key) 50 | end 51 | write(fmt, newname, value[key]) 52 | end 53 | if #value > 0 then 54 | local s = write_array(value) 55 | if not string.empty(s) then 56 | fmt("%s='%s'", name, s) 57 | end 58 | end 59 | end 60 | end 61 | 62 | local function write_patches(w, pkg) 63 | local patches = pkg.patches 64 | if not patches then return end 65 | 66 | local function write_var(name, value) 67 | return write_pkg_var(w, 'patches_', name, value) 68 | end 69 | local names = sort(table.keys(patches)) 70 | 71 | for name in each(names) do 72 | write_var(name, patches[name]) 73 | end 74 | 75 | local resolved = filter(function(item) return item[3] end, patches) 76 | if #resolved > 0 then 77 | w('jagen_stage_apply_patches() {') 78 | for i, item in ipairs(resolved) do 79 | local name, n, path = item[1], item[2], item[3] 80 | w(' pkg_run_patch %d "%s"', n, path) 81 | end 82 | w('}') 83 | end 84 | end 85 | 86 | local function write_files(w, pkg) 87 | local files = pkg.files 88 | if not files then return end 89 | 90 | local function write_var(name, value) 91 | return write_pkg_var(w, 'files_', name, value) 92 | end 93 | 94 | for i = 1, #files do 95 | local item = files[i] 96 | if item._src_path then 97 | write_var(i, { item[1], item._src_path, item.path }) 98 | end 99 | end 100 | end 101 | 102 | function P:write(pkg, filename, engine) 103 | local skip = { stage = true, import = true, export = true, env = true } 104 | 105 | local properties = {} 106 | for key, val in pairs(pkg) do 107 | if key:sub(1, 1) ~= '_' and type(val) == 'string' then 108 | append(properties, key) 109 | skip[key] = true 110 | end 111 | end 112 | table.sort(properties) 113 | 114 | local standard_sections = { 115 | source = true, toolchain = true, build = true, install = true, 116 | 'source', 'toolchain', 'build', 'install' 117 | } 118 | local other_sections = {} 119 | 120 | for key, val in pairs(pkg) do 121 | if not skip[key] 122 | and not standard_sections[key] 123 | and key:sub(1, 1) ~= '_' then 124 | append(other_sections, key) 125 | end 126 | end 127 | 128 | local main, exports, imports, unset, env = {}, {}, {}, {}, {} 129 | 130 | local function add_main(s, ...) 131 | append(main, string.format(s, ...)) 132 | end 133 | local function add_export(s, ...) 134 | append(exports, string.format(s, ...)) 135 | end 136 | local function add_import(s, ...) 137 | append(imports, string.format(s, ...)) 138 | end 139 | local function add_env(s, ...) 140 | append(env, string.format(s, ...)) 141 | end 142 | 143 | for key in each(properties) do 144 | write(add_main, key, pkg[key]) 145 | end 146 | 147 | for name in each(standard_sections) do 148 | if pkg[name] ~= nil then 149 | write(add_main, name, pkg[name]) 150 | end 151 | end 152 | 153 | for name in each(other_sections) do 154 | write(add_main, name, pkg[name]) 155 | end 156 | 157 | local stages = table.keys(pkg.stage) 158 | for stage in each(stages) do 159 | write(add_main, 'stage_'..stage, pkg.stage[stage]) 160 | end 161 | 162 | local function collect_env(env) 163 | for key, value in pairs(env) do 164 | if value == '' then 165 | append(unset, string.format('unset %s', key)) 166 | else 167 | write(add_env, key, value) 168 | end 169 | end 170 | end 171 | 172 | for name in each(pkg.uses) do 173 | local use = engine.packages[name] 174 | if use and use.export and use.export.env then 175 | collect_env(use.export.env) 176 | end 177 | end 178 | 179 | if pkg.env ~= nil then 180 | collect_env(pkg.env) 181 | end 182 | 183 | if pkg.export ~= nil then 184 | write(add_export, '', pkg.export) 185 | end 186 | if pkg.import ~= nil then 187 | write(add_import, '', pkg.import) 188 | end 189 | 190 | table.sort(exports) 191 | table.sort(imports) 192 | table.sort(unset) 193 | table.sort(env) 194 | 195 | local prefix = 'pkg_' 196 | if pkg.name == 'jagen' then 197 | prefix = 'jagen_' 198 | end 199 | 200 | for i = 1, #main do 201 | main[i] = prefix..main[i] 202 | end 203 | for i = 1, #exports do 204 | exports[i] = 'pkg_export_'..exports[i] 205 | end 206 | 207 | function prepend_export(list) 208 | for i = 1, #list do 209 | list[i] = 'export '..list[i] 210 | end 211 | end 212 | 213 | for list in each { main, exports, imports, env } do 214 | prepend_export(list) 215 | end 216 | 217 | local file = assert(io.open(filename, 'w+')) 218 | file:write(table.concat(main, '\n'), '\n') 219 | -- if next(exports) then 220 | -- file:write('\n', table.concat(exports, '\n'), '\n') 221 | -- end 222 | -- if next(imports) then 223 | -- file:write('\n', table.concat(imports, '\n'), '\n') 224 | -- end 225 | -- if next(unset) then 226 | -- file:write('\n', table.concat(unset, '\n'), '\n') 227 | -- end 228 | if next(env) then 229 | file:write('\n', table.concat(env, '\n'), '\n') 230 | end 231 | file:close() 232 | end 233 | 234 | return P 235 | -------------------------------------------------------------------------------- /src/System.lua: -------------------------------------------------------------------------------- 1 | local Command = require 'Command' 2 | local P = {} 3 | 4 | local Log = require 'Log' 5 | 6 | function P.quote(...) 7 | local function quote(arg) 8 | return string.format('%q', tostring(arg)) 9 | end 10 | return table.concat(table.imap({...}, quote), ' ') 11 | end 12 | 13 | function P.expand(str) 14 | local command = string.format("printf '%%s' %q", str) 15 | local file = assert(io.popen(command)) 16 | local output = file:read('*a') 17 | file:close() 18 | return output 19 | end 20 | 21 | function P.mkpath(...) 22 | local sep = '/' 23 | local path = {} 24 | for i = 1, select('#', ...) do 25 | local arg = select(i, ...) 26 | if arg == nil then error("bad argument #"..i.." to 'mkpath' (string expected, got nil)") end 27 | if #arg > 0 then 28 | table.insert(path, arg) 29 | end 30 | end 31 | return table.concat(path, sep) 32 | end 33 | 34 | function P.exec(cmdline, ...) 35 | Log.debug1(cmdline, ...) 36 | local command = string.format(cmdline, ...) 37 | local status = os.execute(command) 38 | -- handling API change in Lua 5.2 39 | if type(status) == 'number' then 40 | return status == 0, status % 0xFF 41 | else 42 | return status or false 43 | end 44 | end 45 | 46 | function P.popen(cmdline, ...) 47 | Log.debug1(cmdline, ...) 48 | local prog = string.format(cmdline, ...) 49 | return assert(io.popen(prog)) 50 | end 51 | 52 | function P.pread(format, cmdline, ...) 53 | local file = P.popen(cmdline, ...) 54 | local out = file:read(format) 55 | file:close() 56 | return out 57 | end 58 | 59 | function P.getenv(vars) 60 | local o = {} 61 | for _, v in ipairs(vars) do 62 | local value = os.getenv(v) 63 | assert(value and #value > 0, 64 | string.format("the environment variable '%s' is not set", v)) 65 | table.insert(o, value) 66 | end 67 | return o 68 | end 69 | 70 | function P.rmrf(...) 71 | if select('#', ...) > 0 then 72 | return Command:new('rm -rf', quote(...)):exec() 73 | end 74 | return true 75 | end 76 | 77 | function P.create_file(path) 78 | if not io.open(path) then 79 | assert(io.open(path, 'w')):close() 80 | end 81 | end 82 | 83 | function P.mkdir(...) 84 | return P.exec('mkdir -p %s', P.quote(...)) 85 | end 86 | 87 | function P.exists(path) 88 | return P.exec('test -e "%s"', path) 89 | end 90 | 91 | function P.file_exists(path) 92 | local file = io.open(path, 'r+') 93 | if file then file:close() end 94 | return file ~= nil 95 | end 96 | 97 | function P.dir_exists(path) 98 | return P.exec('test -d "%s"', path) 99 | end 100 | 101 | -- Returns true if both arguments are existing directories which have the same 102 | -- physical path. 103 | function P.same_dir(dir1, dir2) 104 | -- `cd ""` (with empty dir) does nothing which may lead to surprising 105 | -- result if empty string is supplied as an argument 106 | return Command:newf([[dir1="%s"; dir2="%s" 107 | dir1="$([ "$dir1" ] && cd "$dir1" 2>&- && pwd -P)" 108 | dir2="$([ "$dir2" ] && cd "$dir2" 2>&- && pwd -P)" 109 | test "$dir1" -a "$dir2" && test "$dir1" = "$dir2"]], 110 | assert(dir1), assert(dir2)):exec() 111 | end 112 | 113 | -- Returns true if dir2 is the subdirectory of dir1 114 | function P.is_subdir(dir1, dir2) 115 | return Command:newf([[dir1="%s" dir2="%s" 116 | if [ -d "$dir1" ]; then dir1=$(cd "$dir1" 2>&- && pwd -P); fi 117 | if [ -d "$dir2" ]; then dir2=$(cd "$dir2" 2>&- && pwd -P); fi 118 | test "$dir1" -a "$dir2" && test "$dir2" != "${dir2#$dir1}"]], 119 | assert(dir1), assert(dir2)):exec() 120 | end 121 | 122 | -- Returns true if 'dir' is an existing directory and does not have the same 123 | -- physical path as other directory or the other directory does not exist. 124 | function P.can_delete_safely(dir, other_dir) 125 | return Command:newf([[dir1="%s"; dir2="%s" 126 | dir1="$([ "$dir1" ] && cd "$dir1" 2>&- && pwd -P)" 127 | dir2="$([ "$dir2" ] && cd "$dir2" 2>&- && pwd -P)" 128 | test "$dir1" && test "$dir1" != "$dir2"]], 129 | assert(dir), other_dir or ''):exec() 130 | end 131 | 132 | function P.is_empty(path) 133 | return Command:new('cd', quote(assert(path)), '2>/dev/null', '&&', 'ls -A'):read() == nil 134 | end 135 | 136 | function P.dirname(path) 137 | return P.pread('*l', 'dirname "%s"', path) 138 | end 139 | 140 | function P.write_file(filename, contents) 141 | local file = assert(io.open(filename, 'w')) 142 | file:write(contents) 143 | file:close() 144 | end 145 | 146 | return P 147 | -------------------------------------------------------------------------------- /src/Target.lua: -------------------------------------------------------------------------------- 1 | local Command = require 'Command' 2 | 3 | local Target = {} 4 | 5 | function Target:new(o) 6 | o = o or {} 7 | setmetatable(o, self) 8 | self.__index = self 9 | return o 10 | end 11 | 12 | function Target.from_args(name, stage, config) 13 | local target = Target:new { 14 | name = name, 15 | stage = stage, 16 | config = config, 17 | } 18 | target.ref = tostring(target) 19 | return target 20 | end 21 | 22 | function Target:from_arg(arg) 23 | local name, stage, config 24 | local c = string.split(arg, ':') 25 | 26 | if c[1] and #c[1] > 0 then 27 | name = c[1] 28 | end 29 | if c[2] and #c[2] > 0 then 30 | stage = c[2] 31 | end 32 | if c[3] and #c[3] > 0 then 33 | config = c[3] 34 | end 35 | 36 | return Target.from_args(name, stage, config) 37 | end 38 | 39 | function Target.from_use(spec) 40 | local function parse(spec) 41 | local spec, alias = unpack(spec:split(' as ')) 42 | local name, config = unpack(string.split2(spec, ':')) 43 | local target = Target.from_args(name, nil, config) 44 | if not string.empty(alias) then 45 | target.alias = alias 46 | end 47 | return target 48 | end 49 | if type(spec) == 'string' then 50 | return parse(spec) 51 | elseif type(spec) == 'table' then 52 | local use = parse(spec[1]) 53 | if spec.as then 54 | use.alias = spec.as 55 | elseif type(spec[2]) == 'string' then 56 | use.alias = spec[2] 57 | end 58 | return use 59 | else 60 | error('invalid use specification: '..pretty(spec)) 61 | end 62 | end 63 | 64 | function Target:to_stage() 65 | if self.config then 66 | return string.format('%s:%s', self.stage, self.config) 67 | else 68 | return self.stage 69 | end 70 | end 71 | 72 | function Target:__lt(other) 73 | return self.name < other.name and 74 | self.stage < other.stage and 75 | self.config < other.config 76 | end 77 | 78 | function Target:__eq(other) 79 | return self.name == other.name and 80 | self.stage == other.stage and 81 | self.config == other.config 82 | end 83 | 84 | function Target:__tostring(sep) 85 | local o = {} 86 | sep = sep or ':' 87 | if self.name then table.insert(o, self.name) end 88 | if self.stage then table.insert(o, self.stage) end 89 | if self.config then table.insert(o, self.config) end 90 | return table.concat(o, sep) 91 | end 92 | 93 | function Target:__rawtostring() 94 | local saved = Target.__tostring 95 | Target.__tostring = nil 96 | local s = tostring(self) 97 | Target.__tostring = saved 98 | return s 99 | end 100 | 101 | function Target:match(pattern) 102 | return string.match(tostring(self), pattern) 103 | end 104 | 105 | function Target:touch() 106 | return Command:new('cd "$jagen_build_dir" && touch', quote(self)):exec() 107 | end 108 | 109 | function Target:remove() 110 | return Command:new('cd "$jagen_build_dir" && rm -f', quote(self)):exec() 111 | end 112 | 113 | function Target:log_filename() 114 | return string.format('%s.log', tostring(self)) 115 | 116 | end 117 | 118 | return Target 119 | -------------------------------------------------------------------------------- /src/cmd.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | . "$jagen_dir/src/cprint.sh" 4 | 5 | EOF=$(printf \\034) 6 | S=$(printf '\t') 7 | 8 | mode='' 9 | 10 | die() { 11 | unset IFS 12 | printf "jagen${mode:+ $mode}: %s\n" "$*" 13 | exit 1 14 | } 15 | 16 | join() { 17 | local sep="$1"; 18 | printf -- "%s" "$2" 19 | shift 2 20 | if [ $# -gt 0 ]; then 21 | printf -- "${sep}%s" "$@" 22 | fi 23 | } 24 | 25 | 26 | assert_ninja_found() { 27 | if [ -z "$(command -v ninja)" ]; then 28 | die "could not find 'ninja' command in your PATH, please install \ 29 | Ninja (https://ninja-build.org) to run the build system." 30 | fi 31 | } 32 | 33 | cmd_build() { 34 | . "${jagen_include_dir:?}/jagen.export.sh" 35 | 36 | local IFS="$S" 37 | local build_all no_rebuild follow_selected follow_all print_all is_quiet 38 | local targets log logs err tries follow_pid pipe 39 | local build_log="${jagen_log_dir:?}/build.log" 40 | local outfile="${jagen_log_dir:?}/output.txt" 41 | local c='c'; [ -t 1 ] || c='cs' 42 | 43 | assert_ninja_found 44 | 45 | mkdir -p "$jagen_log_dir" 46 | 47 | while [ $# -gt 0 ]; do 48 | case $1 in 49 | --all) build_all=1 ;; 50 | --no-rebuild) no_rebuild=1 ;; 51 | --follow) follow_selected=1 ;; 52 | --follow-all) follow_all=1 ;; 53 | --progress) print_all=1 ;; 54 | --quiet) is_quiet=1 ;; 55 | --exclude) export jagen__force_exclude=1 ;; 56 | --clean-ignored) export jagen__clean_ignored=1 ;; 57 | --ignore-dirty) export jagen__ignore_dirty=1 ;; 58 | --ignore-exclude) export jagen__ignore_exclude=1 ;; 59 | -*) ;; # ignore unknown options 60 | *) targets="${targets}${S}${1}" 61 | logs="${logs}${S}${jagen_log_dir}/${1}.log" ;; 62 | esac 63 | shift 64 | done 65 | targets=${targets#$S}; logs=${logs#$S} 66 | 67 | export jagen_cmd_targets="$targets" 68 | export jagen__cmd_failed_targets_file="$jagen_build_dir/.build-failed-targets" 69 | 70 | cd "$jagen_build_dir" || return 71 | 72 | : > "$jagen__cmd_failed_targets_file" || return 73 | : > "$build_log" || return 74 | for log in $logs; do 75 | : > "$log" || return 76 | done 77 | 78 | if [ "$build_all" ]; then 79 | targets= 80 | fi 81 | 82 | if [ -z "$no_rebuild" ]; then 83 | rm -f $targets 84 | fi 85 | 86 | # do nothing on CTRL-C, Ninja will still receive it and exit by itself with 87 | # an appropriate status, we will proceed with the specific handling 88 | trap : INT 89 | 90 | if [ "$is_quiet" -o "$follow_selected" -o "$follow_all" ]; then 91 | export jagen__cmd_quiet=1 92 | elif [ "$print_all" ]; then 93 | export jagen__cmd_verbose=1 94 | fi 95 | 96 | if [ ! "$is_quiet" ]; then 97 | if [ "$follow_selected" -o "$follow_all" ]; then 98 | pipe=$(mktemp -u) && mkfifo "$pipe" || return 99 | # tee exits by itself when pipe closes after we kill tail 100 | tee "$outfile" <"$pipe" & 101 | fi 102 | if [ "$follow_selected" ]; then 103 | tail -qFc0 "$build_log" $logs >"$pipe" 2>/dev/null & 104 | follow_pid=$! 105 | elif [ "$follow_all" ]; then 106 | tail -qFc0 "$jagen_log_dir"/*.log >"$pipe" 2>/dev/null & 107 | follow_pid=$! 108 | fi 109 | if [ "$pipe" ]; then 110 | rm "$pipe" 111 | fi 112 | fi 113 | 114 | if [ "$follow_pid" ]; then 115 | # capture Ninja messages to a file to pass it through tail, otherwise 116 | # it writes to the console in parallel and mangles the output 117 | ninja $targets > "$build_log" 2>&1; err=$? 118 | 119 | # Ninja returns 2 when interrupted and discards a buffered output, we 120 | # assume the user wants the command line immediately, so no wait 121 | if [ $err != 2 ]; then 122 | # in the case when the final message is from Ninja we need to 123 | # ensure its log file ends with the EOF marker too, otherwise the 124 | # following loop will wait for nothing 125 | printf $EOF >> "$build_log" 126 | 127 | # when monitoring multiple files a write can end with the EOF 128 | # marker at the exact moment of the check while there is still more 129 | # data pending; we consider a probability of this negligibly small 130 | tries=30 # 3 seconds 131 | until [ "$(tail -c1 "$outfile")" = $EOF ]; do 132 | tries=$((tries-1)); [ $tries = 0 ] && break 133 | sleep 0.1 134 | done 135 | fi 136 | 137 | kill $follow_pid 138 | # waiting with redirecting stderr here allows to get rid of a 139 | # "Terminated: ..." message from Bash 140 | wait $follow_pid 2>/dev/null 141 | 142 | # this could happen if the terminal is very slow (WSL?), just in case 143 | # notify the user that the output is not complete 144 | if [ "$tries" = 0 ]; then 145 | # even after tail is killed the console might be still flushing and 146 | # the warning will be scrolled up out of the screen, a small delay 147 | # might help with that 148 | sleep 0.25 149 | ${c}println "\n{*yellow*}--{~} jagen: timed out while waiting for the console to flush, the output might be truncated" 150 | fi 151 | else 152 | ninja $targets; err=$? 153 | fi 154 | 155 | if [ -s "$jagen__cmd_failed_targets_file" ]; then 156 | local targets= num= limit=1000 157 | while read target log; do 158 | targets="${targets}${S}${target}" 159 | num=$(cat "$log" | wc -l) 160 | if [ $num -gt 0 ]; then 161 | ${c}println "\n{*red*}--{~} jagen: {*red*}failed target{~}: {*white*}$target" 162 | if [ $num -lt $limit ]; then 163 | ${c}println "{*white*}--{~} $log" 164 | cat "$log" 165 | elif [ $num -ge $limit ]; then 166 | ${c}println "{*white*}--{~} the last $limit lines of $log" 167 | # +1 because it counts a EOF marker as a line 168 | tail -n $((limit+1)) "$log" 169 | fi 170 | ${c}println "{*white*}--{~} EOF $log" 171 | fi 172 | done < "$jagen__cmd_failed_targets_file" 173 | targets=${targets#$S} 174 | set -- $targets 175 | if [ $# = 1 ]; then 176 | ${c}println "\n{*red*}--{~} jagen: build stopped: {*red*}target failed:{~} {*white*}$targets" 177 | elif [ $# -gt 1 ]; then 178 | ${c}println "\n{*red*}--{~} jagen: build stopped: {*red*}$# targets failed:{~} {*white*}$(join '{~}, {*white*}' $targets){~}" 179 | fi 180 | fi 181 | 182 | return $err 183 | } 184 | 185 | cmd_image() { 186 | . "$jagen_dir/src/common.sh" || return 187 | local image_script="$(find_in_path "image.sh")" 188 | [ "$image_script" ] || 189 | die "the current project does not support image creation" 190 | "${jagen_shell:-/bin/sh}" "$image_script" "$@" 191 | } 192 | 193 | cmd_find_in_path() { 194 | local arg='' path='' result='' 195 | . "${jagen_dir:?}/src/common.sh" || return 196 | set -- "$@" 197 | for arg; do 198 | path=$(find_in_path "$arg") 199 | if [ -z "$path" ]; then path="$arg"; fi 200 | result="${result}${jagen_S}${path}" 201 | done 202 | result=${result#$jagen_S} 203 | printf '%s' "$result" 204 | } 205 | 206 | echo_if_exists() { 207 | if [ -f "$1" ]; then 208 | echo "$1" 209 | fi 210 | } 211 | 212 | cmd_find_for_refresh() { 213 | local IFS="$jagen_S" paths='' dir 214 | for dir in bin lib src; do 215 | paths="${paths}${jagen_S}${jagen_dir}/${dir}" 216 | done 217 | paths="${paths}${jagen_S}${jagen_include_dir:?}" 218 | paths="${paths#${jagen_S}}" 219 | find $paths -type f -o -type d 220 | echo "$jagen_root_dir" 221 | echo_if_exists "$jagen_root_dir/env.sh" 222 | echo_if_exists "$jagen_root_dir/jagen" 223 | echo_if_exists "$jagen_root_dir/rules.lua" 224 | } 225 | 226 | cmd_get_path() { 227 | . "$jagen_dir/src/common.sh" || return 228 | jagen__get_path 229 | } 230 | 231 | mode="${1:?}" 232 | shift 233 | 234 | case $mode in 235 | build) 236 | cmd_build "$@" 237 | ;; 238 | image) 239 | cmd_image "$@" 240 | ;; 241 | find_for_refresh) 242 | cmd_find_for_refresh "$@" 243 | ;; 244 | get_path) 245 | cmd_get_path "$@" 246 | ;; 247 | *) 248 | die "unknown wrapper command: $1" 249 | ;; 250 | esac 251 | -------------------------------------------------------------------------------- /src/command/build.lua: -------------------------------------------------------------------------------- 1 | local Command = require 'Command' 2 | local Engine = require 'Engine' 3 | local Log = require 'Log' 4 | local Options = require 'Options' 5 | local System = require 'System' 6 | local Target = require 'Target' 7 | 8 | local P = {} 9 | 10 | local function write_targets(targets, args, build_targets_file) 11 | local has_console = os.getenv('jagen__has_console') 12 | local is_interactive = has_console and not args['quiet'] and not (args['progress'] or args['follow'] or args['follow-all']) 13 | local curr_list, saved_list, is_eq = {}, {}, true 14 | 15 | if is_interactive then 16 | for target, explicit in pairs(targets) do 17 | if explicit then 18 | append(curr_list, target) 19 | end 20 | end 21 | sort(curr_list) 22 | end 23 | 24 | local file = io.open(build_targets_file) 25 | if file then 26 | for line in file:lines() do 27 | append(saved_list, line) 28 | end 29 | file:close() 30 | end 31 | 32 | if #curr_list ~= #saved_list then 33 | is_eq = false 34 | else 35 | for i = 1, #curr_list do 36 | if curr_list[i] ~= saved_list[i] then 37 | is_eq = false 38 | break 39 | end 40 | end 41 | end 42 | 43 | if not is_eq then 44 | local file = assert(io.open(build_targets_file, 'w')) 45 | for target in each(curr_list) do 46 | file:write(target, '\n') 47 | end 48 | file:close() 49 | end 50 | end 51 | 52 | function P:run(args) 53 | local options = Options:new { 54 | { 'help,h' }, 55 | { 'match,m' }, 56 | { 'clean,c' }, 57 | { 'clean-ignored,C' }, 58 | { 'all,a' }, 59 | { 'no-rebuild,n' }, 60 | { 'progress,p' }, 61 | { 'follow,f' }, 62 | { 'follow-all,F' }, 63 | { 'quiet,q' }, 64 | { 'exclude,x' }, 65 | { 'ignore-dirty,y' }, 66 | { 'ignore-exclude,Y' }, 67 | { 'no-auto' } 68 | } 69 | args = options:parse(args) 70 | if not args then return false end 71 | if args['help'] then 72 | return Jagen.command['help']({ 'build' }, args['help'].short) 73 | end 74 | 75 | local engine = Engine:new() 76 | local packages = engine:load_rules() 77 | -- if not Engine:validate() then 78 | -- Log.error('aborting the build due to rule errors') 79 | -- return false 80 | -- end 81 | 82 | local targets, arg_clean = {}, args['clean'] or args['clean-ignored'] 83 | 84 | if #args == 0 then 85 | if not (args['progress'] or args['follow'] or args['follow-all']) then 86 | args['quiet'] = true 87 | end 88 | -- for name in kvpairs(packages) do 89 | -- append(args, name) 90 | -- end 91 | end 92 | 93 | local function match(s, p) 94 | return s == 'clean' and p == '^clean$' or 95 | s ~= 'clean' and (not p or p and s:match(p)) 96 | end 97 | 98 | local not_found_args = {} 99 | for arg in each(args) do 100 | local found = false 101 | local parts = arg:split(':') 102 | local name_pat, stage_pat, config_pat = unpack(map(string.to_pattern, parts)) 103 | 104 | local function by_name(pkg) 105 | return match(pkg.name, name_pat) 106 | end 107 | 108 | for name, pkg in filter(by_name)(kvpairs(packages)) do 109 | for stage in kvpairs(pkg.stage or {}) do 110 | local target = Target.from_args(name, stage) 111 | if arg_clean and stage == 'clean' then 112 | targets[tostring(target)] = true 113 | elseif match(stage, stage_pat) then 114 | targets[tostring(target)] = true 115 | found = true 116 | end 117 | end 118 | if not found then 119 | append(not_found_args, arg) 120 | end 121 | end 122 | end 123 | 124 | for arg in each(not_found_args) do 125 | Log.warning('could not find targets matching: %s', arg) 126 | end 127 | 128 | if args['match'] then 129 | local keys = table.keys(targets) 130 | table.sort(keys) 131 | for key in each(keys) do 132 | local explicit = targets[key] 133 | print(string.format('%s%s', key, not explicit and ' *' or '')) 134 | end 135 | return true 136 | end 137 | 138 | -- some targets were specified but none matched, consider this an error 139 | if #args > 0 and not next(targets) then 140 | return false 141 | end 142 | 143 | local config = engine.packages.jagen 144 | local cmd = assert(config.self.cmd) 145 | 146 | write_targets(targets, args, assert(config.build_targets_file)) 147 | 148 | local args_path = System.mkpath(assert(config.build_dir), '.build-args') 149 | local args_file = assert(io.open(args_path, 'w')) 150 | if args._args then 151 | args_file:write(table.concat(args._args, '\n')) 152 | end 153 | args_file:close() 154 | 155 | local ok = Command:new(quote(cmd), 'build', tostring(args), unpack(table.keys(targets))):exec() 156 | 157 | -- io.open(args_path, 'w'):close() 158 | 159 | return ok 160 | end 161 | 162 | return P 163 | -------------------------------------------------------------------------------- /src/command/source.lua: -------------------------------------------------------------------------------- 1 | local Engine = require 'Engine' 2 | local Options = require 'Options' 3 | local Log = require 'Log' 4 | local System = require 'System' 5 | local Target = require 'Target' 6 | 7 | local P = {} 8 | 9 | local function scm_packages(patterns) 10 | local engine = Engine:new() 11 | local packages = engine:load_rules() 12 | local o = {} 13 | 14 | if patterns and #patterns > 0 then 15 | for pattern in each(patterns) do 16 | local cpattern, found = string.convert_pattern(pattern), false 17 | for name, pkg in kvpairs(packages) do 18 | if name:match(cpattern) and pkg.source.scm then 19 | table.insert(o, pkg) found = true 20 | end 21 | end 22 | if not found then 23 | error(string.format('could not find source packages matching: %s', pattern)) 24 | end 25 | end 26 | else 27 | for _, pkg in kvpairs(packages) do 28 | if pkg.source and pkg.source.scm then 29 | table.insert(o, pkg) 30 | end 31 | end 32 | end 33 | 34 | table.sort(o, function (a, b) 35 | return a.name < b.name 36 | end) 37 | 38 | return o 39 | end 40 | 41 | function P:dirty(args) 42 | local packages = scm_packages(args) 43 | for _, pkg in ipairs(packages) do 44 | local source = pkg.source 45 | if System.exists(source.dir) and not System.is_empty(source.dir) and 46 | source:dirty() then 47 | return true 48 | end 49 | end 50 | return false 51 | end 52 | 53 | function P:status(args) 54 | local packages = scm_packages(args) 55 | for _, pkg in ipairs(packages) do 56 | local source = pkg.source 57 | if System.exists(source.dir) and System.exists(source:getscmdir()) then 58 | local head = source:head_name() 59 | local dirty = source:dirty() and ' dirty' or '' 60 | if #dirty > 0 and source.ignore_dirty then 61 | dirty = string.format(' dirty(ignored:%s)', tostring(source.ignore_dirty)) 62 | end 63 | local exclude = source.exclude and ' excluded' or '' 64 | print(string.format("%s%s%s%s%s", pkg.name, 65 | source.location and ' ('..source.location..')', 66 | head and ' ['..head..']', dirty, exclude)) 67 | else 68 | print(string.format("%s (%s): not cloned", pkg.name, source.location)) 69 | end 70 | end 71 | return true 72 | end 73 | 74 | function P:update(args) 75 | local options = Options:new { 76 | { 'ignore-dirty,y' }, 77 | { 'ignore-exclude,Y' } 78 | } 79 | args = options:parse(args) 80 | if not args then return false end 81 | 82 | local packages, ok = scm_packages(args), true 83 | local offline = false -- Jagen.flag 'offline' 84 | local force_exclude = os.getenv('jagen__force_exclude') 85 | local ignore_dirty = args['ignore-dirty'] or os.getenv('jagen__ignore_dirty') 86 | local ignore_exclude = args['ignore-exclude'] or os.getenv('jagen__ignore_exclude') 87 | 88 | function update(source) 89 | local old_head = source:head() 90 | local ok = source:update() 91 | if ok and source:head() ~= old_head then 92 | assert(Target.from_args(assert(source.name), 'unpack'):touch()) 93 | end 94 | return ok 95 | end 96 | 97 | -- Sorting from the shortest to the longest is needed for the case when the 98 | -- source directories are specified inside each other and we need to clone 99 | -- both: deeper one is cloned first, then clone complains about already 100 | -- existing non-empty directory. 101 | table.sort(packages, function (a, b) 102 | return (a.source.dir or '') < (b.source.dir or '') 103 | end) 104 | 105 | for pkg in each(packages) do 106 | local source = pkg.source 107 | local dir = System.expand(source.dir) 108 | if not source:exists() then 109 | if offline then 110 | Log.warning("not cloning %s: offline mode", pkg.name) 111 | ok = false 112 | else 113 | if not source.location then 114 | Log.message("not cloning %s: the source location is not specified", pkg.name) 115 | else 116 | Log.message('cloning %s from %s', pkg.name, source.location) 117 | if not source:clone() then 118 | Log.warning('failed to clone %s from %s to %s', pkg.name, source.location, dir) 119 | ok = false 120 | end 121 | end 122 | end 123 | else 124 | if force_exclude or not ignore_exclude and source.exclude then 125 | Log.message("not updating %s: the source is excluded", pkg.name) 126 | elseif source:dirty() then 127 | ignore_dirty = ignore_dirty and 'forced' or source.ignore_dirty 128 | if ignore_dirty then 129 | Log.message("updating %s: ignoring dirty status of '%s' (%s)", pkg.name, dir, tostring(ignore_dirty)) 130 | ok = update(source) 131 | else 132 | Log.warning("not updating %s: the source directory '%s' has unsaved changes", pkg.name, dir) 133 | ok = false 134 | end 135 | else 136 | Log.message("updating %s: %s", pkg.name, dir) 137 | ok = update(source) 138 | end 139 | end 140 | 141 | if source:exists() then 142 | if not source:fixup() then 143 | Log.warning('failed to fix up %s in %s', pkg.name, dir) 144 | ok = false 145 | end 146 | end 147 | end 148 | 149 | return ok 150 | end 151 | 152 | function P:clean(args) 153 | local options = Options:new { 154 | { 'ignored,i' }, 155 | { 'ignore-dirty,y' }, 156 | { 'ignore-exclude,Y' } 157 | } 158 | args = options:parse(args) 159 | if not args then return false end 160 | 161 | local packages, ok = scm_packages(args), true 162 | local force_exclude = os.getenv('jagen__force_exclude') 163 | local clean_ignored = args['ignored'] or os.getenv('jagen__clean_ignored') 164 | local ignore_dirty = args['ignore-dirty'] or os.getenv('jagen__ignore_dirty') 165 | local ignore_exclude = args['ignore-exclude'] or os.getenv('jagen__ignore_exclude') 166 | 167 | for pkg in each(packages) do 168 | local source = pkg.source 169 | local dir = System.expand(source.dir) 170 | local willclean, reason, comment 171 | if force_exclude or not ignore_exclude and source.exclude then 172 | reason = "the source is excluded" 173 | if force_exclude then 174 | comment = 'forced by the command argument' 175 | end 176 | elseif source:exists() then 177 | if source:dirty() then 178 | if ignore_dirty then 179 | comment = 'forced by the command argument' 180 | end 181 | ignore_dirty = ignore_dirty or source.ignore_dirty 182 | if ignore_dirty then 183 | willclean = true 184 | reason = 'a dirty status is ignored' 185 | if type(ignore_dirty) == 'string' then 186 | comment = ignore_dirty 187 | end 188 | else 189 | reason = "the source directory has unsaved changes (dirty)" 190 | end 191 | else 192 | willclean = true 193 | if source.exclude then 194 | reason = 'an exclude is ignored' 195 | if ignore_exclude then 196 | comment = 'forced by the command argument' 197 | end 198 | end 199 | end 200 | else 201 | reason = 'the source directory does not exist' 202 | end 203 | 204 | if not source.exclude and not force_exclude then 205 | clean_ignored = true 206 | end 207 | 208 | local message = 'cleaning' 209 | if willclean then 210 | if clean_ignored then 211 | message = 'completely '..message 212 | end 213 | else 214 | message = 'not '..message 215 | end 216 | message = message..' '..dir 217 | if reason then 218 | message = message..': '..reason 219 | end 220 | if comment then 221 | message = message..' ('..comment..')' 222 | end 223 | 224 | Log.message(message) 225 | 226 | if willclean then 227 | if source:clean(clean_ignored) then 228 | assert(Target.from_args(assert(pkg.name), 'clean'):touch()) 229 | else 230 | ok = false 231 | end 232 | end 233 | end 234 | 235 | return ok 236 | end 237 | 238 | function P:delete(args) 239 | local packages = scm_packages(args) 240 | for _, pkg in ipairs(packages) do 241 | local source = pkg.source 242 | if System.exists(source.dir) then 243 | if not System.rmrf(source.dir) then 244 | die('failed to delete %s source directory %s', 245 | pkg.name, source.dir) 246 | end 247 | end 248 | end 249 | end 250 | 251 | function P:each(args) 252 | local options = Options:new { 253 | { 'help,h' }, 254 | { 'type=' }, 255 | } 256 | args = options:parse(args) 257 | if not args then 258 | return 22 259 | end 260 | if args['help'] then 261 | return Jagen.command['help'] { 'src_each' } 262 | end 263 | local packages = scm_packages() 264 | for pkg in each(packages) do 265 | local arg_type, src_type = args['type'], pkg.source.type 266 | if arg_type and not Source:is_known(arg_type) then 267 | die('unknown source type: %s', arg_type) 268 | end 269 | if #args < 1 then 270 | die('the command is not specified') 271 | end 272 | local cmd = table.concat(args, ' ') 273 | if not arg_type then 274 | local bin = string.match(cmd, '^(%w+)') 275 | if Source:is_known(bin) then 276 | arg_type = bin 277 | end 278 | end 279 | local dir = System.expand(pkg.source.dir) 280 | local cmd = string.format('cd "%s" && %s', dir, cmd) 281 | if not arg_type or arg_type == src_type then 282 | Log.message('%s: %s', pkg.name, dir) 283 | local ok, status = Command:new(cmd):exec() 284 | if not ok then return status end 285 | end 286 | end 287 | end 288 | 289 | return P 290 | -------------------------------------------------------------------------------- /src/compile.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | pkg_compile() { 4 | local IFS="$jagen_IFS" S="$jagen_FS" A= MA="$(cat "$jagen_build_args_file" 2>&-)" 5 | 6 | local is_offline= verbose_opt= jobs= 7 | local makefile= 8 | 9 | if in_flags offline; then 10 | is_offline=1 11 | fi 12 | 13 | case $pkg_build_type in 14 | gnu) 15 | pkg_run make "$@" $MA 16 | ;; 17 | cmake) 18 | pkg_run "${pkg_build_cmake_executable:?}" --build . -- $(pkg__get_cmake_args) "$@" $MA 19 | ;; 20 | make|kbuild) 21 | if [ -f "${pkg_source_dir:?}/GNUmakefile" ]; then 22 | makefile="$pkg_source_dir/GNUmakefile" 23 | elif [ -f "${pkg_source_dir:?}/makefile" ]; then 24 | makefile="$pkg_source_dir/makefile" 25 | elif [ -f "${pkg_source_dir:?}/Makefile" ]; then 26 | makefile="$pkg_source_dir/Makefile" 27 | fi 28 | if [ "$makefile" ]; then 29 | A="$A${S}-f$makefile" 30 | fi 31 | pkg_run make $A $pkg_build_options "$@" $MA 32 | ;; 33 | linux-kernel) 34 | pkg_run cd "${pkg_source_dir:?}" 35 | if [ "$pkg_build_image" ]; then 36 | pkg_run make $pkg_build_image $MA 37 | fi 38 | pkg_run make modules $MA 39 | ;; 40 | linux-module) 41 | pkg_run make $pkg_build_options "$@" $MA 42 | ;; 43 | executable) 44 | local exe="$jagen_dist_dir/$pkg_source_filename" 45 | if ! [ -f "$exe" ]; then 46 | die "require to run $exe for build but the file was not found" 47 | fi 48 | pkg_run chmod +x "$exe" 49 | pkg_run "$exe" $pkg_build_options "$@" $MA 50 | ;; 51 | rust) 52 | if [ "$CARGO_HOME" ]; then 53 | PATH="$CARGO_HOME/bin:$PATH" 54 | fi 55 | export CARGO_TARGET_DIR="$pkg_build_dir" 56 | cd "${pkg_source_dir:?}" 57 | pkg_is_release && A="--release" 58 | if [ "$pkg_build_rust_toolchain" = 'system' ]; then 59 | debug2 "using cargo from $pkg_build_rust_toolchain: $(which cargo)" 60 | pkg_run cargo build ${pkg_build_system:+--target=$pkg_build_system} \ 61 | $A "$@" $MA 62 | else 63 | debug2 "using rustup from $pkg_build_rust_toolchain: $(which rustup)" 64 | pkg_run rustup run "${pkg_build_rust_toolchain:?}" \ 65 | cargo build ${pkg_build_system:+--target=$pkg_build_system} \ 66 | $A "$@" $MA 67 | fi 68 | ;; 69 | android-gradle) 70 | if ! [ -f "${pkg_source_dir:?}/gradlew" ]; then 71 | die "failed to find Gradle wrapper (gradlew) script in the project's source directory: $pkg_source_dir" 72 | fi 73 | if pkg_is_release; then 74 | A="assembleRelease" 75 | else 76 | A="assembleDebug" 77 | fi 78 | pkg_run bash "${pkg_source_dir:?}/gradlew" $A "$@" $MA 79 | ;; 80 | android-standalone-toolchain) 81 | require toolchain 82 | toolchain_install_android_standalone 83 | ;; 84 | esac 85 | } 86 | -------------------------------------------------------------------------------- /src/configure.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | pkg_configure() { 4 | if [ -z "$pkg_source_dir" ]; then 5 | message "pkg_source_dir is not set, skipping configure" 6 | return 0 7 | fi 8 | 9 | local IFS="$jagen_IFS" S="$jagen_FS" A= MA="$(cat "${jagen_build_args_file:?}" 2>&-)" 10 | local option 11 | 12 | case $pkg_build_type in 13 | gnu) 14 | pkg_run "${pkg_build_configure_file:-$pkg_source_dir/configure}" $A \ 15 | ${pkg_build_system:+--host="$pkg_build_system"} \ 16 | --prefix="$pkg_install_prefix" \ 17 | --disable-dependency-tracking \ 18 | ${pkg_install_root:+--with-sysroot="$pkg_install_root"} \ 19 | $pkg_build_options "$@" $MA 20 | 21 | # Never add RPATH to generated binaries because libtool uses 22 | # various heuristics to determine when to add it, some Linux 23 | # distributions patch it to adjust a list of 'system' paths, but 24 | # generally things seems to work because everyone install to 25 | # /usr/local/lib or /usr/lib or lib64 whatever and these are 26 | # handled specially. Embedded systems often have different 27 | # conventions and naming schemes, libtool not always does the 28 | # 'right' thing and you might end up with a mixed bag of libraries 29 | # some having RPATH and some not. 30 | 31 | if [ -x "./libtool" ]; then 32 | pkg_run sed -i 's|\(hardcode_into_libs\)=yes|\1=no|g' \ 33 | "./libtool" 34 | fi 35 | 36 | ;; 37 | cmake) 38 | if ! [ -f "$pkg_source_dir/CMakeLists.txt" ]; then 39 | die "CMake build type specified but no CMakeLists.txt was found in $pkg_source_dir" 40 | fi 41 | 42 | for option in $pkg_build_options; do 43 | A="$A$S$(eval echo $option)" 44 | done 45 | 46 | pkg_run "${pkg_build_cmake_executable:?}" \ 47 | -G"${pkg_build_cmake_generator:?}" \ 48 | --no-warn-unused-cli \ 49 | $A "$@" $MA "$pkg_source_dir" 50 | ;; 51 | linux-kernel) 52 | pkg_run cd "$pkg_source_dir" 53 | pkg_run make "${pkg_build_config:?build.config is not set}" 54 | pkg_run make prepare 55 | ;; 56 | *) 57 | ;; 58 | esac 59 | } 60 | -------------------------------------------------------------------------------- /src/cprint.sh: -------------------------------------------------------------------------------- 1 | # detecting if the standard output is connected to a terminal does not work 2 | # from a subshell so we need to use separate functions 3 | 4 | cprint_gsub() { 5 | [ $# = 0 ] && return 6 | echo "$@{reset}" | sed \ 7 | -e 's#{reset}#\\033[0m#g' -e 's#{~}#\\033[0m#g' \ 8 | -e 's#{bold}#\\033[1m#g' -e 's#{@}#\\033[1m#g' \ 9 | -e 's#{dim}#\\033[2m#g' -e 's#{h}#\\033[2m#g' \ 10 | -e 's#{italic}#\\033[3m#g' -e 's#{i}#\\033[3m#g' \ 11 | -e 's#{underline}#\\033[4m#g' -e 's#{u}#\\033[4m#g' \ 12 | -e 's#{normal}#\\033[22m#g' -e 's#{n}#\\033[22m#g' \ 13 | -e 's#{black}#\\033[30m#g' -e 's#{k}#\\033[30m#g' \ 14 | -e 's#{red}#\\033[31m#g' -e 's#{r}#\\033[31m#g' \ 15 | -e 's#{green}#\\033[32m#g' -e 's#{g}#\\033[32m#g' \ 16 | -e 's#{yellow}#\\033[33m#g' -e 's#{y}#\\033[33m#g' \ 17 | -e 's#{blue}#\\033[34m#g' -e 's#{b}#\\033[34m#g' \ 18 | -e 's#{magenta}#\\033[35m#g' -e 's#{m}#\\033[35m#g' \ 19 | -e 's#{cyan}#\\033[36m#g' -e 's#{c}#\\033[36m#g' \ 20 | -e 's#{white}#\\033[37m#g' -e 's#{w}#\\033[37m#g' \ 21 | -e 's#{blackbg}#\\033[40m#g' -e 's#{K}#\\033[40m#g' \ 22 | -e 's#{redbg}#\\033[41m#g' -e 's#{R}#\\033[41m#g' \ 23 | -e 's#{greenbg}#\\033[42m#g' -e 's#{G}#\\033[42m#g' \ 24 | -e 's#{yellowbg}#\\033[43m#g' -e 's#{Y}#\\033[43m#g' \ 25 | -e 's#{bluebg}#\\033[44m#g' -e 's#{B}#\\033[44m#g' \ 26 | -e 's#{magentabg}#\\033[45m#g' -e 's#{M}#\\033[45m#g' \ 27 | -e 's#{cyanbg}#\\033[46m#g' -e 's#{C}#\\033[46m#g' \ 28 | -e 's#{whitebg}#\\033[47m#g' -e 's#{W}#\\033[47m#g' \ 29 | -e 's#{defbg}#\\033[49m#g' -e 's#{D}#\\033[49m#g' \ 30 | -e 's#{\*black\*}#\\033[90m#g' -e 's#{\*k\*}#\\033[90m#g' \ 31 | -e 's#{\*red\*}#\\033[91m#g' -e 's#{\*r\*}#\\033[91m#g' \ 32 | -e 's#{\*green\*}#\\033[92m#g' -e 's#{\*g\*}#\\033[92m#g' \ 33 | -e 's#{\*yellow\*}#\\033[93m#g' -e 's#{\*y\*}#\\033[93m#g' \ 34 | -e 's#{\*blue\*}#\\033[94m#g' -e 's#{\*b\*}#\\033[94m#g' \ 35 | -e 's#{\*magenta\*}#\\033[95m#g' -e 's#{\*m\*}#\\033[95m#g' \ 36 | -e 's#{\*cyan\*}#\\033[96m#g' -e 's#{\*c\*}#\\033[96m#g' \ 37 | -e 's#{\*white\*}#\\033[97m#g' -e 's#{\*w\*}#\\033[97m#g' \ 38 | -e 's#{\*blackbg\*}#\\033[100m#g' -e 's#{\*K\*}#\\033[100m#g' \ 39 | -e 's#{\*redbg\*}#\\033[101m#g' -e 's#{\*R\*}#\\033[101m#g' \ 40 | -e 's#{\*greenbg\*}#\\033[102m#g' -e 's#{\*G\*}#\\033[102m#g' \ 41 | -e 's#{\*yellowbg\*}#\\033[103m#g' -e 's#{\*Y\*}#\\033[103m#g' \ 42 | -e 's#{\*bluebg\*}#\\033[104m#g' -e 's#{\*B\*}#\\033[104m#g' \ 43 | -e 's#{\*magentabg\*}#\\033[105m#g' -e 's#{\*M\*}#\\033[105m#g' \ 44 | -e 's#{\*cyanbg\*}#\\033[106m#g' -e 's#{\*C\*}#\\033[106m#g' \ 45 | -e 's#{\*whitebg\*}#\\033[107m#g' -e 's#{\*W\*}#\\033[107m#g' 46 | } 47 | 48 | csprint_gsub() { 49 | [ $# = 0 ] && return 50 | echo "$@" | sed \ 51 | -e 's#{reset}##g' -e 's#{~}##g' \ 52 | -e 's#{bold}##g' -e 's#{@}##g' \ 53 | -e 's#{dim}##g' -e 's#{h}##g' \ 54 | -e 's#{italic}##g' -e 's#{i}##g' \ 55 | -e 's#{underline}##g' -e 's#{u}##g' \ 56 | -e 's#{normal}##g' -e 's#{n}##g' \ 57 | -e 's#{black}##g' -e 's#{k}##g' \ 58 | -e 's#{red}##g' -e 's#{r}##g' \ 59 | -e 's#{green}##g' -e 's#{g}##g' \ 60 | -e 's#{yellow}##g' -e 's#{y}##g' \ 61 | -e 's#{blue}##g' -e 's#{b}##g' \ 62 | -e 's#{magenta}##g' -e 's#{m}##g' \ 63 | -e 's#{cyan}##g' -e 's#{c}##g' \ 64 | -e 's#{white}##g' -e 's#{w}##g' \ 65 | -e 's#{blackbg}##g' -e 's#{K}##g' \ 66 | -e 's#{redbg}##g' -e 's#{R}##g' \ 67 | -e 's#{greenbg}##g' -e 's#{G}##g' \ 68 | -e 's#{yellowbg}##g' -e 's#{Y}##g' \ 69 | -e 's#{bluebg}##g' -e 's#{B}##g' \ 70 | -e 's#{magentabg}##g' -e 's#{M}##g' \ 71 | -e 's#{cyanbg}##g' -e 's#{C}##g' \ 72 | -e 's#{whitebg}##g' -e 's#{W}##g' \ 73 | -e 's#{defbg}##g' -e 's#{D}##g' \ 74 | -e 's#{\*black\*}##g' -e 's#{\*k\*}##g' \ 75 | -e 's#{\*red\*}##g' -e 's#{\*r\*}##g' \ 76 | -e 's#{\*green\*}##g' -e 's#{\*g\*}##g' \ 77 | -e 's#{\*yellow\*}##g' -e 's#{\*y\*}##g' \ 78 | -e 's#{\*blue\*}##g' -e 's#{\*b\*}##g' \ 79 | -e 's#{\*magenta\*}##g' -e 's#{\*m\*}##g' \ 80 | -e 's#{\*cyan\*}##g' -e 's#{\*c\*}##g' \ 81 | -e 's#{\*white\*}##g' -e 's#{\*w\*}##g' \ 82 | -e 's#{\*blackbg\*}##g' -e 's#{\*K\*}##g' \ 83 | -e 's#{\*redbg\*}##g' -e 's#{\*R\*}##g' \ 84 | -e 's#{\*greenbg\*}##g' -e 's#{\*G\*}##g' \ 85 | -e 's#{\*yellowbg\*}##g' -e 's#{\*Y\*}##g' \ 86 | -e 's#{\*bluebg\*}##g' -e 's#{\*B\*}##g' \ 87 | -e 's#{\*magentabg\*}##g' -e 's#{\*M\*}##g' \ 88 | -e 's#{\*cyanbg\*}##g' -e 's#{\*C\*}##g' \ 89 | -e 's#{\*whitebg\*}##g' -e 's#{\*W\*}##g' 90 | } 91 | 92 | cprint() { 93 | local O 94 | # prevent command substitution from stripping ending newlines from the 95 | # result by padding it with a character 96 | O=$(cprint_gsub "$@\034") 97 | # trim the added character to keep the arguments verbatim 98 | printf -- "${O%\034}" 99 | } 100 | 101 | csprint() { 102 | local O="$(csprint_gsub "$@\034")" 103 | printf -- "${O%\034}" 104 | } 105 | 106 | cprintln() { 107 | # if a reset is not performed before the final newline it can lead to 108 | # graphical artifacts such as leaving the cursor with the wrong color or 109 | # the last background color leaking from the previous line 110 | cprint "$@{reset}\n" 111 | } 112 | 113 | csprintln() { 114 | # no reset beacuse "sc" variant just strips control sequences anyway 115 | csprint "$@\n" 116 | } 117 | -------------------------------------------------------------------------------- /src/help.lua: -------------------------------------------------------------------------------- 1 | local usage = [[ 2 | Usage: jagen [OPTIONS...] 3 | 4 | Generates and manages a build system according to the predefined rules. 5 | 6 | COMMANDS 7 | 8 | help Show jagen usage information 9 | clean Clean up build root 10 | refresh Regenerate the build system 11 | build Build or rebuild the specified targets 12 | source Manage SCM package sources 13 | list List various information about the current project 14 | update Update the specified layers or Jagen itself 15 | 16 | Use 'jagen help ' to get help about individual commands. 17 | 18 | ]] 19 | 20 | local help = [[ 21 | Usage: jagen help [section] 22 | jagen --help [section] 23 | jagen
help 24 | jagen
--help 25 | 26 | Shows usage information about jagen commands or other help sections. 27 | 28 | SYNOPSIS 29 | 30 | If a command was specified before or after help argument its usage will be 31 | shown, if there are other help section available with the given name - this 32 | section will be shown, or general usage information will be shown if nothing 33 | was specified. 34 | 35 | It is also possible to use '-h' instead of '--help'. 36 | 37 | ]] 38 | 39 | local clean = [[ 40 | Usage: jagen clean [package[:config]...] 41 | 42 | Deletes package build directories or all generated files and directories 43 | inside the current build root. 44 | 45 | OPTIONS 46 | 47 | -i, --ignored also remove ignored files 48 | -x, --exclude treat all sources as excluded 49 | -y, --ignore-dirty ignore dirty status of source directories 50 | -Y, --ignore-exclude do not skip excluded packages 51 | 52 | SYNOPSIS 53 | 54 | There can be multiple arguments in the form of: or :. 55 | Build directories of given packages will be removed. If is supplied 56 | the corresponding build directory will be removed, if only is supplied 57 | all build directories will be removed. 58 | 59 | If no arguments are given, than everything in the current build root is 60 | cleaned up. The following directories are recursively deleted: 61 | 62 | jagen_build_dir 63 | jagen_include_dir 64 | jagen_log_dir 65 | 66 | Actual paths depend on configuration. After the deletion regenerates the 67 | build system using the 'jagen refresh' command. 68 | 69 | ]] 70 | 71 | local refresh = [[ 72 | Usage: jagen refresh 73 | 74 | Regenerates the build system from rules according to configuration. 75 | 76 | ]] 77 | 78 | local build = { 79 | usage = [[ 80 | Usage: jagen build [OPTION...] [PATTERN...] [--] [TOOL OPTIONS...] 81 | 82 | Builds or rebuilds the specified targets. 83 | 84 | OPTIONS 85 | 86 | -h, --help print this help message 87 | -m, --match print expanded value of target patterns and exit 88 | -c, --clean clean package's build directories before the build 89 | -C, --clean-ignored also clean ignored files 90 | -a, --all continue until everything is up to date 91 | -n, --no-rebuild do not rebuild targets which are already up to date 92 | -p, --progress print the output of build targets after completion 93 | -f, --follow follow a build output for the specified targets only 94 | -F, --follow-all follow all build output 95 | -q, --quiet inhibit build output 96 | -x, --exclude treat all sources as excluded 97 | -y, --ignore-dirty ignore dirty status of source directories 98 | -Y, --ignore-exclude do not skip excluded packages 99 | 100 | Use the command 'jagen help targets' for information about targets. 101 | 102 | ]], synopsis = [[ 103 | SYNOPSIS 104 | 105 | The specified patterns are expanded and matching targets are rebuilt. Use the 106 | '--match' option to print the matches without building anything. 107 | 108 | Use the '--clean' option to remove the package's build directories before the 109 | start. It also causes the 'configure' stage of the affected packages to 110 | become out of date. 111 | 112 | Use the '--all' option to build everything out of date in the current project 113 | in addition to the specified targets. 114 | 115 | The '--no-rebuild' option causes the command to behave similarly to 'make': 116 | it ensures that targets are up to date instead of rebuilding them 117 | unconditionally. 118 | 119 | The '--progress' option prints the buffered output of all build targets as 120 | soon as they complete. 121 | 122 | The '--follow' option allows monitoring the output from build commands in real 123 | time. It shows the output only for targets specified for the current build 124 | command. Works best when used for a single package or dependent targets 125 | because when there are several package builds in progress their output will be 126 | intermixed. 127 | 128 | The '--follow-all' option shows the output from all currently running build 129 | commands at the same time. This includes the targets building as dependencies 130 | of the ones specified as build command arguments and all others not currently 131 | up to date if used in combination with '--all' option. 132 | 133 | With the '--quiet' option the output from commands is collected to logs but 134 | not displayed on the console during the execution unless the stage finished 135 | with an error. 136 | 137 | Arguments after '--' will be passed literally to the underlying build tool. 138 | The handling depends on the build tool in question but be aware of the case 139 | when multiple targets matched as the same arguments will be passed to all of 140 | their build commands which might have surprising results. 141 | 142 | ]] 143 | } 144 | 145 | local targets = [[ 146 | 147 | Targets are specified as '::'. Available package stages 148 | are filtered with the given expression. Omitted component means 'all'. For 149 | example: 150 | 151 | utils - select all stages of the utils package 152 | utils:install - select all utils install stages 153 | utils::host - select all host utils stages 154 | utils:compile:host - select only host utils compile stage 155 | :build:host - select host build stages of all packages 156 | 157 | When a target is succesfully built the stamp file is created in the build 158 | directory with the name: ____. This file is used to 159 | determine if the target is up to date. Deleting it will cause the 160 | corresponding target to be rebuilt unconditionally next time the build system 161 | runs. 162 | 163 | ]] 164 | 165 | local source = [[ 166 | Usage: jagen source [PACKAGES...] 167 | 168 | Manage SCM package sources. 169 | 170 | SYNOPSIS 171 | 172 | The optional PACKAGES argument should be the list of SCM packages defined in 173 | the current environment. If none are specified, all are assumed. 174 | 175 | COMMANDS 176 | 177 | dirty Check if packages source directories have any changes 178 | status Show packages location, head commit and dirty status 179 | clean Clean up packages source directories 180 | update Update the sources to the latest upstream version 181 | clone Clone the specified packages 182 | delete Delete packages source directories 183 | each Execute Shell command inside each source directory 184 | 185 | The 'dirty' command exits with 0 (true) status if any of the specified 186 | packages source directories have changes. It exits with 1 (false) status if 187 | all sources are clean. Intended for usage by shell scripts. 188 | 189 | The 'status' command prints SCM packages status in human readable form. 190 | 191 | The 'clean' command resets modifications to the HEAD state and deletes 192 | all extra files in packages source directories. 193 | 194 | The 'update' command fetches the latest sources from upstream and tries to 195 | merge them. It does nothing if there are modifications in the working 196 | directory (dirty returns true); commit, stash or clean changes before issuing 197 | the 'update' command in this case. 198 | 199 | The 'clone' command clones the specified packages. 200 | 201 | The 'delete' command deletes packages source directories. 202 | 203 | Run `jagen source each --help` to see the reference for the 'each' subcommand. 204 | 205 | ]] 206 | 207 | local src_each = [[ 208 | Usage: jagen source each 209 | 210 | Execute Shell command for each source directory. 211 | 212 | SYNOPSIS 213 | 214 | The 'each' subcommand concatenates its arguments and executes the result as 215 | Shell command inside the source directories. Use the '--type' argument to 216 | filter the source directories by type, e.g. '--type git' will run the command 217 | only for 'git' sources. If the type is not specified but the command starts 218 | with one of the known source types (git, hg and repo) then it will be set 219 | implicitly, otherwise the command will be run for all source directories. 220 | 221 | EXAMPLES 222 | 223 | Run `git status` for sources of type 'git': 224 | 225 | jagen source each git status 226 | 227 | Run `ls` for all source directories: 228 | 229 | jagen source each ls 230 | 231 | Run `ls` only for 'hg' sources: 232 | 233 | jagen source each --type hg ls 234 | 235 | ]] 236 | 237 | local image = [[ 238 | Usage: jagen image 239 | 240 | Creates and manages filesystem images. 241 | 242 | COMMANDS 243 | 244 | Subcommands are project-specific. 245 | 246 | ]] 247 | 248 | local list = [[ 249 | Usage: jagen list [packages|layers] [OPTIONS...] 250 | 251 | Lists various information about the current project. 252 | 253 | COMMANDS 254 | 255 | packages List package rules and their origin 256 | 257 | The 'packages' command displays a list of all currently defined packages 258 | along with contexts where their definitions were found. Contexts could be 259 | rule files or other packages which mention given package as their requires. 260 | In the displayed filenames the parent directory of the current project is 261 | shown as '...'. 262 | 263 | packages options: 264 | 265 | --all, -a 266 | Show also implicit rules added by the generator such as the toolchain 267 | dependencies. These rules will be marked with "*". 268 | 269 | --depth,-d 270 | Set the maximum depth of the rule contexts displayed. If none was 271 | specified the 0 is the default which results in showing only the 272 | toplevel packages explicitly defined in the rule files. If the option 273 | is given without a value it is set to 999 which means show all 274 | contexts. 275 | 276 | layers Show currently defined layers and their file paths. 277 | 278 | ]] 279 | 280 | local update = [[ 281 | Usage: jagen update [|jagen|self]... 282 | 283 | Updates the specified layers or Jagen itself. 284 | 285 | SYNOPSIS 286 | 287 | Specify a list of shell-like patterns of layer names to update. To see all 288 | currently defined layers use the `jagen list layers` command. If nothing is 289 | specified then all layers will be updated. 290 | 291 | Special names 'jagen' and 'self' can be added to the list to also update the 292 | Jagen repository associated with the project. These special names do not 293 | participate in the layer name pattern matching and should be specified 294 | explicitly. 295 | 296 | EXAMPLES 297 | 298 | To update all currently defined layers: 299 | 300 | jagen update 301 | 302 | To update only Jagen: 303 | 304 | jagen update self 305 | 306 | To update all layers and Jagen: 307 | 308 | jagen update self '*' 309 | 310 | To update only layers with names starting with 'ja': 311 | 312 | jagen update 'ja*' 313 | 314 | ]] 315 | 316 | return { 317 | usage = usage, 318 | help = help, 319 | clean = clean, 320 | refresh = refresh, 321 | build = build, 322 | source = source, 323 | src = source, 324 | src_each = src_each, 325 | image = image, 326 | list = list, 327 | update = update, 328 | targets = targets 329 | } 330 | -------------------------------------------------------------------------------- /src/image.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | pkg__image() { 4 | case $pkg_build_type in 5 | linux-kernel) 6 | # use_env kbuild 7 | pkg_run cd "$pkg_source_dir" 8 | pkg_run make "${pkg_build_image:?}" 9 | pkg_run install -vm644 \ 10 | "$pkg_build_dir/arch/$pkg_build_arch/boot/${pkg_build_image:?}" \ 11 | "$jagen_build_dir" 12 | ;; 13 | esac 14 | } 15 | -------------------------------------------------------------------------------- /src/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | pkg_install() { 4 | local IFS="$jagen_IFS" MA="$(cat "${jagen_build_args_file:?}" 2>&-)" 5 | local pkg_install_type="${pkg_install_type:-$pkg_build_type}" 6 | 7 | case $pkg_install_type in 8 | gnu|make|kbuild) 9 | pkg_run make \ 10 | ${pkg_install_root:+DESTDIR="$pkg_install_root"} \ 11 | $pkg_install_args "$@" $MA install 12 | 13 | for name in $pkg_install_libs; do 14 | pkg_fix_pc "$name" 15 | # pkg_fix_la "$pkg_install_root$pkg_install_prefix/lib/lib${name}.la" "$pkg_install_root" 16 | 17 | if [ -z "$pkg_install_config_script" ]; then 18 | pkg_install_config_script="/bin/${pkg_name}-config" 19 | fi 20 | pkg_fix_config_script "${pkg_install_dir:?}${pkg_install_config_script}" 21 | done 22 | ;; 23 | cmake) 24 | if [ "$pkg_install_root" ]; then 25 | export DESTDIR="$pkg_install_root" 26 | fi 27 | pkg_run "${pkg_build_cmake_executable:?}" --build . --target install -- \ 28 | $(pkg__get_cmake_args) $pkg_install_args "$@" $MA 29 | unset DESTDIR 30 | ;; 31 | linux-kernel) 32 | [ "$pkg_build_arch" ] || 33 | die "unable to install: build arch is not set" 34 | [ "$pkg_build_image" ] || 35 | die "unable to install: build image is not set" 36 | pkg_run cd "${pkg_source_dir:?}" 37 | pkg_run install -vm644 \ 38 | "${pkg_build_dir:?}/arch/${pkg_build_arch:?}/boot/${pkg_build_image:?}" \ 39 | "${jagen_build_dir:?}" 40 | pkg_run make \ 41 | INSTALL_MOD_PATH="${INSTALL_MOD_PATH:-${pkg_install_dir:?}}" \ 42 | $pkg_install_args "$@" $MA modules_install 43 | ;; 44 | linux-module) 45 | pkg_install_modules 46 | ;; 47 | toolchain) 48 | require toolchain 49 | toolchain_generate_wrappers "${pkg_source_dir:?}" 50 | ;; 51 | rust-toolchain) 52 | if ! rustup toolchain list | grep -q "^${pkg_build_name:?}"; then 53 | pkg_run rustup install "$pkg_build_name" || 54 | die "failed to install Rust toolchain: $pkg_build_name" 55 | fi 56 | if [ "$pkg_build_system" ]; then 57 | if ! rustup target list --toolchain "$pkg_build_name" | grep -q \ 58 | -e "^${pkg_build_system}.*(default)" \ 59 | -e "^${pkg_build_system}.*(installed)" 60 | then 61 | pkg_run rustup target add "$pkg_build_system" || 62 | die "failed to add Rust target: $pkg_build_system" 63 | fi 64 | fi 65 | ;; 66 | android-standalone-toolchain) 67 | require toolchain 68 | toolchain_generate_wrappers "${pkg_build_dir:?}" 69 | ;; 70 | esac 71 | 72 | if [ "$pkg_install_ldconfig" ]; then 73 | pkg_run_ldconfig 74 | fi 75 | } 76 | 77 | pkg__modules_install() { 78 | pkg_run make -C "${KERNEL_SRC:?}" M="$PWD/$1" \ 79 | INSTALL_MOD_PATH="${INSTALL_MOD_PATH:-${pkg_install_dir:?}}" \ 80 | modules_install 81 | } 82 | 83 | pkg_install_modules() { 84 | local dir 85 | 86 | if [ "$pkg_install_module_dirs" ]; then 87 | for dir in $pkg_install_module_dirs; do 88 | pkg__modules_install "$dir" 89 | done 90 | else 91 | # NOTE: Passing '.' here causes 'modules_install' to append 92 | # full path to the module sources after install-dir. 93 | pkg__modules_install '' 94 | fi 95 | } 96 | -------------------------------------------------------------------------------- /src/patch.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | pkg_patch() { 4 | local IFS; unset IFS 5 | if is_function jagen_stage_apply_patches; then 6 | if [ "$pkg_source_exclude" ]; then 7 | message "not patching $pkg_name: the source is excluded" 8 | else 9 | jagen_stage_apply_patches 10 | fi 11 | fi 12 | } 13 | 14 | pkg_patch_copy_files() { 15 | local IFS="$jagen_IFS" i=0 name value src dst dir 16 | while :; do 17 | i=$((i+1)) 18 | name="\$pkg_files_$i" 19 | value=$(eval echo \"$name\") 20 | [ "$value" ] || break; 21 | set -- $value 22 | name=$1 src=$2 dst=${3:-$pkg_build_dir/$name} dir=$(dirname "$dst") 23 | [ -d "$dir" ] || pkg_run mkdir -p "$dir" 24 | pkg_run cp -va "$src" "$dst" 25 | done 26 | } 27 | 28 | pkg__generate_autogen() { 29 | message "generating GNU build system for $pkg_name using autogen.sh" 30 | pkg_run sh ./autogen.sh 31 | } 32 | 33 | pkg__generate_autoreconf() { 34 | message "generating GNU build system for $pkg_name using autoreconf" 35 | pkg_run autoreconf -vif 36 | } 37 | 38 | pkg_generate() { 39 | [ "$pkg_source_dir" ] || return 0 40 | case $pkg_build_generate in 41 | autogen|yes) 42 | if [ -f ./autogen.sh ]; then 43 | pkg__generate_autogen 44 | else 45 | pkg__generate_autoreconf 46 | fi ;; 47 | autoreconf) 48 | pkg__generate_autoreconf 49 | ;; 50 | esac 51 | } 52 | -------------------------------------------------------------------------------- /src/stages.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | . "$jagen_dir/env.sh" || { echo "Failed to load environment"; exit 1; } 4 | 5 | include "$jagen__src_dir/util" 6 | include "$jagen__src_dir/unpack" 7 | include "$jagen__src_dir/patch" 8 | include "$jagen__src_dir/configure" 9 | include "$jagen__src_dir/compile" 10 | include "$jagen__src_dir/install" 11 | include "$jagen__src_dir/image" 12 | 13 | jagen_stage_clean() { 14 | pkg_clean 15 | } 16 | 17 | jagen_stage_unpack() { 18 | pkg_unpack 19 | } 20 | 21 | jagen_stage_update() { 22 | pkg_unpack 23 | } 24 | 25 | jagen_stage_patch() { 26 | pkg_patch 27 | pkg_patch_copy_files 28 | pkg_generate 29 | } 30 | 31 | jagen_stage_configure() { 32 | pkg_configure 33 | } 34 | 35 | jagen_stage_compile() { 36 | pkg_compile 37 | } 38 | 39 | jagen_stage_install() { 40 | pkg_install 41 | } 42 | 43 | jagen_stage_image() { 44 | pkg__image 45 | } 46 | -------------------------------------------------------------------------------- /src/toolchain.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #shellcheck disable=2154,2155 3 | 4 | toolchain_cc() { 5 | printf "${1:-${CC:-${pkg_build_system:+${pkg_build_system}-}gcc}}" 6 | } 7 | 8 | toolchain_get_sysroot() { 9 | real_path $("$(toolchain_cc "$1")" --print-sysroot) 10 | } 11 | 12 | toolchain_get_arch_sysroot() { 13 | real_path $("$(toolchain_cc "$1")" $pkg_build_cflags --print-sysroot) 14 | } 15 | 16 | toolchain_find_path() { 17 | local filename="$("$(toolchain_cc "$2")" $pkg_build_cflags --print-file-name="${1:?}")" 18 | if [ "$filename" ]; then 19 | real_path $(dirname "$filename") 20 | fi 21 | } 22 | 23 | toolchain_get_lib_dir() { 24 | toolchain_find_path libstdc++.a "$@" 25 | } 26 | 27 | toolchain_match() { 28 | local path="${1:?}" 29 | local name filename="${path##*/}" prefix="${pkg_export_system:+${pkg_export_system}-}" 30 | local IFS="$jagen_FS" 31 | shift 32 | for name; do 33 | echo F=$filename N=\"$name\" P=\"$prefix\" 34 | case $filename in 35 | ${prefix}${name}-*|${prefix}${name}) 36 | echo match 37 | return 0 ;; 38 | esac 39 | done 40 | return 1 41 | } 42 | 43 | toolchain_find() { 44 | local src_dir="${1:?}" name="${2:?}" path= 45 | # this is a very common layout 46 | path=$(find "$src_dir/bin" -maxdepth 1 -type f -executable \ 47 | -name "$name" 2>/dev/null) 48 | [ "$path" ] && { echo "$path"; return; } 49 | # something not common, try to a deep search 50 | path=$(find "$src_dir" -maxdepth 10 -type f -executable \ 51 | -path '*/bin/*' -name "$name" 2>/dev/null) 52 | [ "$path" ] && { echo "$path"; return; } 53 | # fallback to PATH 54 | path=$(command -v "$path") 55 | echo "$path" 56 | } 57 | 58 | toolchain_wrap() { 59 | local dest_dir="${1:?}" path="${2:?}" varname="${3:?}" 60 | local filename="${path##*/}" cmd= 61 | local wrapper="${dest_dir}/${filename:?}" 62 | message "wrap ${wrapper#$jagen_root_dir/} -> ${path#$jagen_root_dir/}" 63 | case $varname in 64 | cflags|cxxflags) 65 | # compiler is often called as a linker, so it needs ldflags too 66 | cmd="$path \$pkg_toolchain_$varname \$pkg_toolchain_ldflags" ;; 67 | *) 68 | cmd="$path \$pkg_toolchain_$varname" ;; 69 | esac 70 | # if the wrapper is a symlink echo will overwrite the original 71 | # executable in the toolchain unless we remove the link first 72 | rm -f "$wrapper" || return 73 | echo "exec $cmd \"\$@\"" >"$wrapper" || return 74 | chmod +x "$wrapper" || return 75 | } 76 | 77 | toolchain_generate_wrappers() { 78 | local src_dir="${1:?}" dest_dir="${jagen_bin_dir:?}/${pkg_name}" 79 | 80 | [ -d "$src_dir" ] || \ 81 | die "toolchain_generate_wrappers: the src dir '$src_dir' does not exist" 82 | 83 | pkg_run mkdir -p "$dest_dir" 84 | 85 | toolchain_wrap "$dest_dir" $(toolchain_find "$src_dir" "$pkg_export_cc") cflags 86 | toolchain_wrap "$dest_dir" $(toolchain_find "$src_dir" "$pkg_export_cxx") cxxflags 87 | toolchain_wrap "$dest_dir" $(toolchain_find "$src_dir" "$pkg_export_cpp") cflags 88 | toolchain_wrap "$dest_dir" $(toolchain_find "$src_dir" "$pkg_export_ld") ldflags 89 | } 90 | 91 | toolchain_install_runtime() { 92 | local dest_dir="${1:-${jagen_target_dir:?}}/lib" 93 | local sysroot_dir="$(toolchain_get_arch_sysroot)" 94 | local lib_dir="$(toolchain_get_lib_dir)" 95 | local filter_file="$(find_in_path toolchain_libs.txt)" 96 | 97 | : ${sysroot_dir:?} 98 | : ${lib_dir:?} 99 | : ${filter_file:?} 100 | 101 | message "install toolchain runtime: $sysroot_dir, $lib_dir to $dest_dir" 102 | 103 | pkg_run rsync -va --chmod=F0755 \ 104 | --filter="merge $filter_file" \ 105 | "$sysroot_dir/lib/" "$dest_dir" 106 | 107 | pkg_run rsync -va --chmod=F0755 \ 108 | --filter="merge $filter_file" \ 109 | "$lib_dir/" "$dest_dir" 110 | } 111 | 112 | toolchain_install_ldconfig() { 113 | : 114 | } 115 | 116 | toolchain_install_android_standalone() { 117 | unset IFS 118 | local make_script="${toolchain_source_dir:?}/build/tools/make_standalone_toolchain.py" 119 | pkg_run "$make_script" --force \ 120 | --arch "${pkg_build_arch:?}" \ 121 | ${pkg_build_android_api:+--api "$pkg_build_android_api"} \ 122 | --install-dir "${pkg_build_dir:?}" 123 | } 124 | -------------------------------------------------------------------------------- /src/toolchain_libs.txt: -------------------------------------------------------------------------------- 1 | - libcidn* 2 | - libmemusage* 3 | + ld*.so* 4 | + libc*.so* 5 | + libcrypt*.so* 6 | + libdl*.so* 7 | + libgcc_s*.so* 8 | + libm*.so* 9 | + libnsl*.so* 10 | + libnss_dns*.so* 11 | + libnss_files*.so* 12 | + libpthread*.so* 13 | + libresolv*.so* 14 | + librt*.so* 15 | + libstdc++*.so* 16 | + libutil*.so* 17 | - * 18 | -------------------------------------------------------------------------------- /src/unpack.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | pkg__is_scm() { 4 | case $pkg_source_type in 5 | git|hg|repo) return 0 ;; 6 | esac 7 | return 1 8 | } 9 | 10 | pkg__download_cleanup() { 11 | pkg__rm $pkg__current_download 12 | pkg__current_download= 13 | } 14 | 15 | pkg__download() { 16 | local src_path="${1:?}" 17 | local dest_path="${2:?}" 18 | local cookie_path= confirm_key= 19 | 20 | pkg_run mkdir -p "${dest_path%/*}" 21 | 22 | if [ "$pkg_source_type" = "dist:gdrive" ]; then 23 | cookie_path=$(mktemp /tmp/jagen-cookie.XXXXXXXX) 24 | [ "$cookie_path" ] || die "failed to create a temp file for storing cookies" 25 | 26 | pkg__current_download="$cookie_path $dest_path" 27 | pkg__curl -c "$cookie_path" "https://drive.google.com/uc?export=download&id=$src_path" -o "$dest_path" 28 | 29 | confirm_key=$(awk '$1 ~ /#HttpOnly_.drive.google.com/ && $6 ~ /^download_warning_/ { print $NF }' "$cookie_path") 30 | if [ "$confirm_key" ]; then 31 | pkg__curl "https://drive.google.com/uc?export=download&confirm=$confirm_key&id=$src_path" -o "$dest_path" 32 | fi 33 | else 34 | pkg__current_download="$dest_path" 35 | pkg__curl "$src_path" -o "$dest_path" 36 | fi 37 | 38 | # cleanup only cookie if set 39 | pkg__current_download="$cookie_path" 40 | } 41 | 42 | pkg__path_is_uri() { 43 | [ "${1:?}" != "${1#*://}" ] 44 | } 45 | 46 | pkg__uri_is_local() { 47 | [ "${1:?}" != "${1#file://}" ] 48 | } 49 | 50 | pkg__unpack_tag_filename() { 51 | echo "$pkg_work_dir/unpacked" 52 | } 53 | 54 | pkg__get_unpack_tag() { 55 | local file="${1:?}" checksum="${2-}" size= 56 | if [ -z "$checksum" ]; then 57 | size=$(jagen_get_file_size "$file") 58 | fi 59 | echo $(basename "$file"):${checksum:-$size} 60 | } 61 | 62 | pkg__read_unpack_tag() { 63 | local filename=$(pkg__unpack_tag_filename) 64 | if [ -f "$filename" ]; then 65 | cat "$filename" 66 | fi 67 | } 68 | 69 | pkg__write_unpack_tag() { 70 | echo "$(pkg__get_unpack_tag "${1:?}" "$2")" > "$(pkg__unpack_tag_filename)" 71 | } 72 | 73 | pkg__unpack_dist() { 74 | local src_path="${1:?}" dest_dir="${2:?}" dist_type= 75 | local dist_path="${jagen_dist_dir:?}/${pkg_source_filename:?}" 76 | local source_checksum= checksum= 77 | 78 | if ! [ -f "$dist_path" ]; then 79 | if [ "$pkg_source_type" = "dist:gdrive" ] || pkg__path_is_uri "$src_path"; then 80 | if in_flags offline && ! pkg__uri_is_local "$src_path"; then 81 | die "unable to download $src_path: offline mode" 82 | else 83 | pkg__download "$src_path" "$dist_path" 84 | fi 85 | else 86 | die "unable to unpack $dist_path: the file is not found and download location is not specified" 87 | fi 88 | fi 89 | 90 | if [ "$pkg_source_sha256sum" ]; then 91 | source_checksum="$pkg_source_sha256sum" 92 | checksum=$(jagen_get_file_checksum sha256 "$dist_path") 93 | elif [ "$pkg_source_sha1sum" ]; then 94 | source_checksum="$pkg_source_sha1sum" 95 | checksum=$(jagen_get_file_checksum sha1 "$dist_path") 96 | elif [ "$pkg_source_md5sum" ]; then 97 | source_checksum="$pkg_source_md5sum" 98 | checksum=$(jagen_get_file_checksum md5 "$dist_path") 99 | fi 100 | 101 | case $? in 102 | 0) if [ "$source_checksum" != "$checksum" ]; then 103 | die "checksum verification failed for $dist_path: expected '$source_checksum', actual '$checksum'" 104 | fi ;; 105 | 1) die "checksum verification failed for $dist_path" ;; 106 | 2) ;; # maybe make this depend on flag 107 | esac 108 | 109 | if ! is_function jagen_stage_apply_patches && 110 | [ "$(pkg__get_unpack_tag "$dist_path" $checksum)" = "$(pkg__read_unpack_tag)" ]; then 111 | message "already unpacked $dist_path" 112 | return 0 113 | fi 114 | 115 | [ -d "$dest_dir" ] || pkg_run mkdir -p "$dest_dir" 116 | pkg_run cd "$dest_dir" 117 | 118 | dist_type=$(file -b --mime-type "$dist_path") 119 | [ $? = 0 ] || die "failed to find file type of '$dist_path'" 120 | 121 | case $dist_type in 122 | application/x-sharedlib) 123 | pkg_run chmod +x "$dist_path" 124 | return ;; 125 | */zip) 126 | pkg_run unzip -o "$dist_path" 127 | return ;; 128 | esac 129 | 130 | case $pkg_source_filename in 131 | *.tar|*.tar.*|*.tgz|*.tbz2|*.txz) 132 | pkg_run tar -xf "$dist_path" ;; 133 | esac 134 | 135 | pkg__write_unpack_tag "$dist_path" $checksum 136 | } 137 | 138 | pkg_clean() { 139 | local dir toclean= IFS="$jagen_S" 140 | if [ "$pkg_build_clean" ]; then 141 | for dir in $pkg_build_clean; do 142 | toclean="$toclean$jagen_S$dir" 143 | done 144 | elif [ "$pkg_config" ]; then 145 | toclean="$pkg_build_dir" 146 | else 147 | # This works for dist in_source packages only because the source dir is 148 | # inside the work dir in the default configuration. 149 | toclean="$pkg_work_dir${jagen_S}$pkg_source_dir" 150 | fi 151 | if [ "$pkg_stage_clean_command" ]; then 152 | echo $pkg_stage_clean_command 153 | fi 154 | for dir in $toclean; do 155 | if [ -d "$dir" ]; then 156 | if jagen_is_same_dir "$dir" "$pkg_source_dir"; then 157 | if pkg__is_scm; then 158 | pkg_run _jagen src clean "$pkg_name" 159 | else 160 | warning "not removing '$dir' of $pkg_name: it is the source directory" 161 | fi 162 | else 163 | debug "removing $dir" 164 | pkg_run rm -rf "$dir" 165 | fi 166 | fi 167 | done 168 | } 169 | 170 | pkg_unpack() { 171 | local IFS; unset IFS 172 | 173 | case $pkg_source_type in 174 | dist|dist:*) 175 | pkg__unpack_dist "$pkg_source_location" "$pkg_work_dir" 176 | ;; 177 | git|hg|repo) 178 | pkg_run _jagen src update "$pkg_name" 179 | ;; 180 | dir|'') ;; 181 | *) 182 | die "unknown source type: $pkg_source_type" 183 | ;; 184 | esac 185 | } 186 | -------------------------------------------------------------------------------- /src/util.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | pkg__get_jobs() { 4 | printf "%s" "${pkg_build_jobs:-${jagen_jobs:-$(jagen_nproc)}}" 5 | } 6 | 7 | pkg_run() { 8 | local IFS; unset IFS 9 | local cmd="$1" jobs="$(pkg__get_jobs)" 10 | shift 11 | 12 | jagen__need_cmd "$cmd" 13 | 14 | case $cmd in 15 | make) 16 | cmd="$cmd -j$jobs" 17 | [ "$jagen_build_verbose" ] && cmd="$cmd V=1" 18 | ;; 19 | ninja) 20 | cmd="$cmd -j$jobs" 21 | [ "$jagen_build_verbose" ] && cmd="$cmd -v" 22 | ;; 23 | esac 24 | 25 | debug1 $cmd "$*" 26 | $cmd "$@" || exit 27 | } 28 | 29 | pkg__rm() { 30 | local item 31 | for item in "$@"; do 32 | if [ "$item" ]; then 33 | debug "removing $item" 34 | rm -f "$item" 35 | fi 36 | done 37 | } 38 | 39 | pkg__curl() { 40 | jagen__need_cmd curl 41 | pkg_run curl -fL ${jagen_insecure:+--insecure} "$@" 42 | } 43 | 44 | pkg_run_patch() { 45 | local num="${1:?}" filename="${2:?}" out err 46 | message "applying patch '$filename' ($num)" 47 | debug1 patch -r- -N -p"$num" -i "$filename" 48 | out=$(LC_LANG=C patch -r- -N -p"$num" -i "$filename"); err=$? 49 | # print the captured output to make it visible in the logs 50 | echo "$out" 51 | if [ $err = 1 ]; then 52 | # When patch encounters errors it exits with status 1. But we do not 53 | # want to count the case when there were no other problems except 54 | # skipping already applied chunks as an error to allow running patch 55 | # stages more than once. This not always works though. 56 | echo "$out" | awk -vRS='patching ' -vFS='\n' '!/^$/ && ( \ 57 | $1 !~ /^file / || 58 | $2 != "Reversed (or previously applied) patch detected! Skipping patch." || 59 | $3 !~ /^[[:digit:]]+ out of [[:digit:]]+ hunks? ignored$/) \ 60 | { exit 3 }'; err=$? 61 | fi 62 | [ $err = 0 ] || exit $err 63 | } 64 | 65 | pkg_strip_root() { 66 | local root="${1:?}" files 67 | local strip="${pkg_build_toolchain_prefix}strip" 68 | 69 | files=$(find "$root" -type f -not -name "*.ko" \ 70 | "(" -path "*/lib*" -o -path "*/bin*" -o -path "*/sbin*" ")" | \ 71 | xargs -r file | grep "ELF.*\(executable\|shared object\).*not stripped" | cut -d: -f1) 72 | 73 | for f in $files; do 74 | pkg_run "$strip" --strip-unneeded \ 75 | -R .comment \ 76 | -R .GCC.command.line \ 77 | -R .note.gnu.gold-version \ 78 | "$f" 79 | done 80 | } 81 | 82 | # Some packages write full paths with sysroot prepended to their pc files which 83 | # causes the sysroot to be prepended twice in build flags of packages which 84 | # actually support it. Namely fontconfig does this. It is easier to patch 85 | # everything just in case than fixing every individual package. 86 | pkg_fix_pc() { 87 | local name="${1:?}" 88 | local filename="$pkg_install_dir/lib/pkgconfig/${name}.pc" 89 | debug1 "fix pc $filename" 90 | if [ -f "$filename" -a "$pkg_install_root" ]; then 91 | pkg_run sed -i "s|$pkg_install_root||g" "$filename" 92 | fi 93 | } 94 | 95 | pkg_fix_la() { 96 | local filename="${1:?}" prefix="$2" 97 | debug1 "fix la $filename $prefix" 98 | if [ "$prefix" ]; then 99 | pkg_run sed -i "s|^\(libdir=\)'\(.*\)'$|\1'${prefix}\2'|" "$filename" 100 | fi 101 | } 102 | 103 | pkg_fix_config_script() { 104 | local filename="${1:?}" 105 | if [ "$pkg_install_root" -a -f "$filename" ]; then 106 | pkg_run sed -E -i "s|^(prefix=)$pkg_install_prefix$|\1$pkg_install_root|" $filename 107 | fi 108 | } 109 | 110 | pkg_run_ldconfig() { 111 | pkg_run ldconfig -n "$pkg_install_dir/lib" 112 | } 113 | 114 | pkg_sync_dirs() { 115 | local source_dir="${1:?}" 116 | local dest_dir="${2:?}" 117 | local filter_file= 118 | 119 | [ -d "$source_dir" ] || 120 | die "Sync source directory '$source_dir' is not exists" 121 | [ -d "$dest_dir" ] || 122 | die "Sync destination directory '$dest_dir' is not exists" 123 | 124 | if [ "$3" ]; then 125 | filter_file=$(find_in_path "$3") 126 | [ "$filter_file" ] || 127 | die "Could not find filter file '$3' for syncronization of '$source_dir' to '$dest_dir'" 128 | fi 129 | 130 | pkg_run rsync -va --delete --delete-excluded \ 131 | ${filter_file:+--filter="merge ${filter_file}"} \ 132 | "$source_dir" "$dest_dir" 133 | } 134 | 135 | pkg_link() { 136 | local target="${1:?}" src="${2:?}" 137 | local dir="$(dirname "$src")" 138 | 139 | pkg_run mkdir -p "$dir" 140 | pkg_run cd "$dir" 141 | pkg_run rm -rf "$(basename "$src")" 142 | pkg_run ln -rs "$target" "$src" 143 | pkg_run cd "$OLDPWD" 144 | } 145 | 146 | pkg_install_file() { 147 | local src="$(find_in_path "${1:?}")" dest="${2:?}" 148 | [ -f "$src" ] || die "failed to find '$1' in path" 149 | pkg_run mkdir -p "$(dirname "$dest")" 150 | pkg_run cp -vf "$src" "$dest" 151 | } 152 | 153 | pkg__fname() { 154 | printf '%s' "${1:?}${2:+:$2}" 155 | } 156 | 157 | pkg__get_cmake_args() { 158 | local args= v_arg= j_arg= 159 | if [ "$jagen_build_verbose" ]; then 160 | case $pkg_build_generator in 161 | *Ninja) v_arg="-v" ;; 162 | *Makefiles) v_arg="VERBOSE=1" ;; 163 | esac 164 | fi 165 | case $pkg_build_generator in 166 | *Makefiles) j_arg="-j$(pkg__get_jobs)" ;; 167 | esac 168 | args="$v_arg $j_arg"; args=${args# }; args=${args% } 169 | printf '%s' "$args" 170 | } 171 | 172 | pkg_get_build_profile() { 173 | local profile="$pkg_build_profile" 174 | case $profile in 175 | release|debug|release_with_debug) 176 | echo $profile ;; 177 | *) if [ "$pkg_build_type" = 'android-gradle' ]; then 178 | echo debug 179 | else 180 | echo "${jagen_build_profile:-release}" 181 | fi ;; 182 | esac 183 | } 184 | 185 | pkg_cmake_build_type() { 186 | local profile="$(pkg_get_build_profile)" 187 | case $profile in 188 | release) 189 | echo "Release" ;; 190 | debug) 191 | echo "Debug" ;; 192 | release_with_debug) 193 | echo "RelWithDebInfo" ;; 194 | *) 195 | echo "Release" ;; 196 | esac 197 | } 198 | 199 | pkg_is_release() { 200 | test "$(pkg_get_build_profile)" = "release" 201 | } 202 | 203 | pkg_is_debug() { 204 | test "$(pkg_get_build_profile)" = "debug" 205 | } 206 | 207 | pkg_is_release_with_debug() { 208 | test "$(pkg_get_build_profile)" = "release_with_debug" 209 | } 210 | --------------------------------------------------------------------------------