├── dub.selections.json ├── test ├── hello.d ├── .gitignore ├── issue15914.d ├── issue15914-bisect.log └── run-tests.sh ├── .gitmodules ├── .travis.yml ├── LICENSE.md ├── appveyor.yml ├── .gitignore ├── common.d ├── dub.sdl ├── bisect.ini.sample ├── config.d ├── repo.d ├── custom.d ├── digger.ini.sample ├── CHANGELOG.md ├── README.md ├── digger.d ├── bisect.d └── install.d /dub.selections.json: -------------------------------------------------------------------------------- 1 | { 2 | "fileVersion": 1, 3 | "versions": { 4 | "ae": "0.0.3236" 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /test/hello.d: -------------------------------------------------------------------------------- 1 | import std.stdio; 2 | 3 | void main() 4 | { 5 | writeln("Hello, world!"); 6 | } 7 | -------------------------------------------------------------------------------- /test/.gitignore: -------------------------------------------------------------------------------- 1 | /*.lst 2 | /bisect.ini 3 | /digger 4 | /digger-web 5 | /digger.ini 6 | /digger.log 7 | /issue15914.o 8 | /work/ 9 | -------------------------------------------------------------------------------- /test/issue15914.d: -------------------------------------------------------------------------------- 1 | import std.getopt; 2 | 3 | void main() 4 | { 5 | bool opt; 6 | string[] args = ["program"]; 7 | getopt(args, config.passThrough, 'a', &opt); 8 | } 9 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "ae"] 2 | path = ae 3 | url = https://github.com/CyberShadow/ae 4 | [submodule "win32"] 5 | path = win32 6 | url = https://github.com/CS-svnmirror/dsource-bindings-win32 7 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: d 2 | d: dmd-2.076.0 3 | os: 4 | - linux 5 | - osx 6 | addons: 7 | apt: 8 | packages: 9 | - gcc-multilib # https://stackoverflow.com/questions/12591629/gcc-cannot-find-bits-predefs-h-on-i686 10 | - g++-multilib 11 | script: test/run-tests.sh 12 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Digger is currently dual-licensed under: 2 | 3 | 1. [Mozilla Public License v2.0](http://mozilla.org/MPL/2.0/) (license used by the ae library) 4 | 5 | 2. [Boost Software License v1.0](http://www.boost.org/LICENSE_1_0.txt) (license used by the D Programming Language) 6 | 7 | [Rationale](https://github.com/CyberShadow/Digger/issues/15). 8 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | install: 2 | - git submodule update --init --recursive 3 | - ps: Start-FileDownload 'http://downloads.dlang.org/releases/2.x/2.076.0/dmd.2.076.0.windows.zip' -FileName 'dmd2.7z' 4 | - 7z x dmd2.7z > nul 5 | - set PATH=%CD%\dmd2\windows\bin;%CD%\dmd2\windows\bin64;%PATH% 6 | - dmd.exe --version 7 | build_script: 8 | - bash test/run-tests.sh 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # User configuration 2 | 3 | /digger.ini 4 | /bisect.ini 5 | /customization-state.json 6 | 7 | # When workDir = . 8 | 9 | /repo/ 10 | /build/ 11 | /current 12 | /result 13 | /cache/ 14 | /cache-git/ 15 | /temp-cache/ 16 | /bootstrap/ 17 | /dl/ 18 | /tmp/ 19 | 20 | # When workDir = ./work 21 | 22 | /work 23 | 24 | # POSIX 25 | 26 | /digger 27 | /digger-web 28 | 29 | # Windows 30 | 31 | *.exe 32 | *.pdb 33 | *.ilk 34 | *.exp 35 | *.lib 36 | *.suo 37 | 38 | # Dub 39 | 40 | /digger_web 41 | /.dub/ 42 | -------------------------------------------------------------------------------- /common.d: -------------------------------------------------------------------------------- 1 | module common; 2 | 3 | import std.stdio; 4 | 5 | import config; 6 | 7 | enum diggerVersion = "3.0.9"; 8 | 9 | /// Send to stderr iff we have a console to write to 10 | void writeToConsole(string s) 11 | { 12 | version (Windows) 13 | { 14 | import core.sys.windows.windows; 15 | auto h = GetStdHandle(STD_ERROR_HANDLE); 16 | if (!h || h == INVALID_HANDLE_VALUE) 17 | return; 18 | } 19 | 20 | stderr.write(s); stderr.flush(); 21 | } 22 | 23 | void log(string s) 24 | { 25 | if (!opts.quiet) 26 | writeToConsole("digger: " ~ s ~ "\n"); 27 | } 28 | -------------------------------------------------------------------------------- /test/issue15914-bisect.log: -------------------------------------------------------------------------------- 1 | digger: f9a65147517ab78d2edd423edd02f3899fa21405 is the first bad commit 2 | commit f9a65147517ab78d2edd423edd02f3899fa21405 3 | Author: H. S. Teoh 4 | Date: Wed Feb 17 22:01:21 2016 -0800 5 | 6 | phobos: Merge pull request #3859 from BBasile/getopt-checker 7 | 8 | https://github.com/dlang/phobos/pull/3859 9 | 10 | Static verification of std.getopt arguments with more helpful error messages 11 | 12 | diff --git a/phobos b/phobos 13 | --- a/phobos 14 | +++ b/phobos 15 | @@ -1 +1 @@ 16 | -Subproject commit 32f032fe65fd6362fcf7542b9b4bc66d531cf321 17 | +Subproject commit 405858d816890edd068e494d4600a3d60b5c1184 18 | digger: Bisection completed successfully. 19 | -------------------------------------------------------------------------------- /dub.sdl: -------------------------------------------------------------------------------- 1 | name "digger" 2 | description "A tool to build D and bisect old D versions" 3 | authors "Vladimir Panteleev " 4 | homepage "https://github.com/CyberShadow/Digger" 5 | license "MPL-2.0" 6 | license "Boost-1.0" 7 | 8 | --------------------------- 9 | 10 | # Main package is the Digger tool. 11 | 12 | targetType "executable" 13 | 14 | # https://github.com/dlang/dub/issues/825 15 | sourceFiles "bisect.d" 16 | sourceFiles "common.d" 17 | sourceFiles "config.d" 18 | sourceFiles "custom.d" 19 | sourceFiles "digger.d" 20 | sourceFiles "install.d" 21 | sourceFiles "repo.d" 22 | 23 | dependency "ae" version="==0.0.3236" 24 | dependency "ae:sys-net-wininet" version="==0.0.3236" platform="windows" 25 | dependency "ae:sys-net-curl" version="==0.0.3236" platform="posix" 26 | 27 | # Apparently needed for LDC. See: 28 | # https://github.com/CyberShadow/Digger/issues/53 29 | libs "zlib" platform="posix" 30 | -------------------------------------------------------------------------------- /bisect.ini.sample: -------------------------------------------------------------------------------- 1 | # This file is a bisect spec example. 2 | # Copy this file to another filename, and make the appropriate 3 | # changes below. 4 | 5 | # Starting points. 6 | # Known bad (with regression) and good (w/o regression) commits. 7 | # Digger will verify these commits to avoid a false conclusion. 8 | # Supported formats: 9 | # - "master" (implies latest commit) 10 | # - SHA-1 hash of D.git repo commit 11 | # - SHA-1 hash of a component's repo commit 12 | # (must be a pull request merge commit) 13 | # - Full GitHub URL of a pull request, e.g.: 14 | # https://github.com/dlang/dmd/pull/123 15 | # - Optionally, "@" followed by a time spec (e.g. "@ 2013-01-02", 16 | # "master @ October 2013", "2.065 @ 15 days ago" or "@ Tuesday") 17 | # or a tag (e.g. "master @ v2.085.0"). 18 | # If no ref is specified, master is assumed. 19 | 20 | bad = master 21 | good = @ 1 month ago 22 | 23 | # Reverse search 24 | # Sometimes, it is useful to know to know which revision fixed 25 | # something. Although this can be accomplished by reversing the 26 | # return value of the test command, the terminology can get 27 | # confusing, since "good" will mean "still broken" and "bad" 28 | # will mean "already fixed". This option causes Digger to 29 | # reverse the test command output, as well as diagnostic 30 | # messages. 31 | 32 | reverse = false 33 | 34 | # Tester command. 35 | # Should exit with status 0 to indicate success (a "good" commit, 36 | # not manifesting the regression), and any other status to 37 | # indicate failure (a "bad" commit, with the regression). 38 | # Exit code 125 is treated specially and indicates that the 39 | # current D revision is untestable, and that Git should try 40 | # another. 41 | # The current bisected version of DMD will be in PATH, 42 | # so there's no need to specify an absolute path to it. 43 | 44 | tester = cd C:\Temp && dmd -o- test.d 45 | 46 | # If you want to bisect a D build failure, enable bisectBuild. 47 | # Instead of executing the tester command, Digger will use 48 | # whether the D build succeeds as the signal for bisection. 49 | # Note that if bisectBuild is enabled, the tester option must 50 | # be unset. 51 | 52 | #bisectBuild = true 53 | 54 | # Similarly, if you'd like to bisect a D test failure. 55 | # Implies bisectBuild. 56 | 57 | #bisectBuildTest = true 58 | 59 | # Test environment. 60 | # Uses the same syntax as build.environment, however it 61 | # is only applied during the execution of the tester 62 | # command specified above. 63 | 64 | # Extra patches to apply at every bisect step. 65 | # Specified in the same way as a build spec for "digger build", 66 | # but without the leading branch name. 67 | 68 | #extraSpec = +dmd#9425 69 | 70 | [environment] 71 | 72 | # Optionally, you can specify additional build options. 73 | # These use the same format, and override those found in digger.ini. 74 | [build] 75 | 76 | components.enable.rdmd = false 77 | -------------------------------------------------------------------------------- /config.d: -------------------------------------------------------------------------------- 1 | module config; 2 | 3 | import std.file; 4 | import std.path; 5 | import std.process : environment; 6 | import std.string; 7 | 8 | import core.runtime; 9 | 10 | import ae.sys.d.manager; 11 | import ae.sys.paths; 12 | import ae.utils.funopt; 13 | import ae.utils.meta; 14 | import ae.utils.sini; 15 | 16 | static import std.getopt; 17 | 18 | struct Opts 19 | { 20 | Option!(string, hiddenOption) dir; 21 | Option!(string, "Path to the configuration file to use", "PATH") configFile; 22 | Switch!("Silence log output", 'q') quiet; 23 | Switch!("Do not update D repositories from GitHub [local.offline]") offline; 24 | Option!(string, "How many jobs to run makefiles in [local.makeJobs]", "N", 'j') jobs; 25 | Option!(string[], "Additional configuration. Equivalent to digger.ini settings.", "NAME=VALUE", 'c', "config") configLines; 26 | 27 | Parameter!(string, "Action to perform (see list below)") action; 28 | Parameter!(immutable(string)[]) actionArguments; 29 | } 30 | immutable Opts opts; 31 | 32 | struct ConfigFile 33 | { 34 | DManager.Config.Build build; 35 | DManager.Config.Local local; 36 | 37 | struct App 38 | { 39 | string[] gitOptions; 40 | } 41 | App app; 42 | } 43 | immutable ConfigFile config; 44 | 45 | shared static this() 46 | { 47 | alias fun = structFun!Opts; 48 | enum funOpts = FunOptConfig([std.getopt.config.stopOnFirstNonOption]); 49 | void usageFun(string) {} 50 | auto opts = funopt!(fun, funOpts, usageFun)(Runtime.args); 51 | 52 | if (opts.dir) 53 | chdir(opts.dir.value); 54 | 55 | enum CONFIG_FILE = "digger.ini"; 56 | 57 | if (!opts.configFile) 58 | { 59 | auto searchDirs = [ 60 | string.init, 61 | thisExePath.dirName, 62 | __FILE__.dirName, 63 | ] ~ getConfigDirs() ~ [ 64 | buildPath(environment.get("HOME", environment.get("USERPROFILE")), ".digger"), // legacy 65 | ]; 66 | version (Posix) 67 | searchDirs ~= "/etc/"; // legacy 68 | 69 | foreach (dir; searchDirs) 70 | { 71 | auto path = dir.buildPath(CONFIG_FILE); 72 | if (path.exists) 73 | { 74 | opts.configFile = path; 75 | break; 76 | } 77 | } 78 | } 79 | 80 | if (opts.configFile.value.exists) 81 | { 82 | config = cast(immutable) 83 | opts.configFile.value 84 | .readText() 85 | .splitLines() 86 | .parseIni!ConfigFile(); 87 | } 88 | 89 | string workDir; 90 | if (!config.local.workDir.length) 91 | if (exists("repo")) // legacy 92 | workDir = "."; 93 | else 94 | workDir = "work"; // TODO use ~/.cache/digger 95 | else 96 | workDir = config.local.workDir.expandTilde(); 97 | config.local.workDir = workDir.absolutePath().buildNormalizedPath(); 98 | 99 | if (opts.offline) 100 | config.local.offline = opts.offline; 101 | if (opts.jobs) 102 | config.local.makeJobs = opts.jobs; 103 | opts.configLines.parseIniInto(config); 104 | 105 | .opts = cast(immutable)opts; 106 | 107 | if (config.app.gitOptions) 108 | { 109 | import ae.sys.git : Repository; 110 | Repository.globalOptions ~= config.app.gitOptions; 111 | } 112 | } 113 | 114 | @property string subDir(string name)() { return buildPath(config.local.workDir, name); } 115 | -------------------------------------------------------------------------------- /repo.d: -------------------------------------------------------------------------------- 1 | module repo; 2 | 3 | import std.array; 4 | import std.algorithm; 5 | import std.exception; 6 | import std.file; 7 | import std.parallelism : parallel; 8 | import std.path; 9 | import std.process; 10 | import std.range; 11 | import std.regex; 12 | import std.string; 13 | 14 | import ae.sys.file; 15 | import ae.sys.d.manager; 16 | import ae.utils.regex; 17 | 18 | import common; 19 | import config : config, opts; 20 | import custom : parseSpec; 21 | 22 | //alias BuildConfig = DManager.Config.Build; 23 | 24 | final class DiggerManager : DManager 25 | { 26 | this() 27 | { 28 | this.config.build = cast().config.build; 29 | this.config.local = cast().config.local; 30 | this.verifyWorkTree = true; // for commands which don't take BuildOptions, like bisect 31 | } 32 | 33 | override void log(string s) 34 | { 35 | common.log(s); 36 | } 37 | 38 | void logProgress(string s) 39 | { 40 | log((" " ~ s ~ " ").center(70, '-')); 41 | } 42 | 43 | override SubmoduleState parseSpec(string spec) 44 | { 45 | return .parseSpec(spec); 46 | } 47 | 48 | override MetaRepository getMetaRepo() 49 | { 50 | if (!repoDir.exists) 51 | log("First run detected.\nPlease be patient, " ~ 52 | "cloning everything might take a few minutes...\n"); 53 | return super.getMetaRepo(); 54 | } 55 | 56 | override string getCallbackCommand() 57 | { 58 | return escapeShellFileName(thisExePath) ~ " do callback"; 59 | } 60 | 61 | string[string] getBaseEnvironment() 62 | { 63 | return d.baseEnvironment.vars; 64 | } 65 | 66 | bool haveUpdate; 67 | 68 | void needUpdate() 69 | { 70 | if (!haveUpdate) 71 | { 72 | d.update(); 73 | haveUpdate = true; 74 | } 75 | } 76 | } 77 | 78 | DiggerManager d; 79 | 80 | static this() 81 | { 82 | d = new DiggerManager(); 83 | } 84 | 85 | string parseRev(string rev) 86 | { 87 | auto args = ["log", "--pretty=format:%H"]; 88 | 89 | d.needUpdate(); 90 | 91 | auto metaRepo = d.getMetaRepo(); 92 | auto repo = &metaRepo.git(); 93 | 94 | // git's approxidate accepts anything, so a disambiguating prefix is required 95 | if (rev.canFind('@') && !rev.canFind("@{")) 96 | { 97 | auto parts = rev.findSplit("@"); 98 | auto at = parts[2].strip(); 99 | 100 | // If this is a named tag, use the date of the tagged commit. 101 | try 102 | { 103 | auto sha1 = metaRepo.getRef("refs/tags/" ~ at); 104 | at = repo.query("log", "-1", "--pretty=format:%cI", sha1); 105 | } 106 | catch (Exception e) {} 107 | 108 | if (at.startsWith("#")) // For the build-all command - skip this many commits 109 | args ~= ["--skip", at[1..$]]; 110 | else 111 | args ~= ["--until", at]; 112 | rev = parts[0].strip(); 113 | } 114 | 115 | if (rev.empty) 116 | rev = "origin/master"; 117 | 118 | try 119 | if (metaRepo.getRef("origin/" ~ rev)) 120 | return repo.query(args ~ ["-n", "1", "origin/" ~ rev]); 121 | catch (Exception e) {} 122 | 123 | try 124 | if (metaRepo.getRef(rev)) 125 | return repo.query(args ~ ["-n", "1", rev]); 126 | catch (Exception e) {} 127 | 128 | if (rev.startsWith("https://github.com")) 129 | { 130 | auto grep = repo.query("log", "-n", "2", "--pretty=format:%H", "--grep", "^" ~ escapeRE(rev), "origin/master").splitLines(); 131 | if (grep.length == 1) 132 | return grep[0]; 133 | } 134 | 135 | auto pickaxe = repo.query("log", "-n", "3", "--pretty=format:%H", "-S" ~ rev, "origin/master").splitLines(); 136 | if (pickaxe.length && pickaxe.length <= 2) // removed <- added 137 | return pickaxe[$-1]; // the one where it was added 138 | 139 | throw new Exception("Unknown/ambiguous revision: " ~ rev); 140 | } 141 | -------------------------------------------------------------------------------- /custom.d: -------------------------------------------------------------------------------- 1 | module custom; 2 | 3 | import std.algorithm; 4 | import std.array; 5 | import std.conv; 6 | import std.exception; 7 | import std.file; 8 | import std.getopt; 9 | import std.path; 10 | import std.stdio; 11 | import std.string; 12 | 13 | import ae.sys.d.manager; 14 | import ae.utils.array; 15 | import ae.utils.json; 16 | import ae.utils.regex; 17 | 18 | import common; 19 | import config : config, subDir; 20 | import install; 21 | import repo; 22 | 23 | alias indexOf = std.string.indexOf; 24 | 25 | // https://issues.dlang.org/show_bug.cgi?id=15777 26 | alias strip = std.string.strip; 27 | 28 | alias subDir!"result" resultDir; 29 | 30 | /// We save a JSON file to the result directory with the build parameters. 31 | struct BuildInfo 32 | { 33 | string diggerVersion; 34 | string spec; 35 | DiggerManager.Config.Build config; 36 | DManager.SubmoduleState components; 37 | } 38 | 39 | enum buildInfoFileName = "build-info.json"; 40 | 41 | void prepareResult() 42 | { 43 | log("Moving..."); 44 | if (resultDir.exists) 45 | resultDir.rmdirRecurse(); 46 | rename(d.buildDir, resultDir); 47 | 48 | log("Build successful.\n\nTo start using it, run `digger install`, or add %s to your PATH.".format( 49 | resultDir.buildPath("bin").absolutePath() 50 | )); 51 | } 52 | 53 | /// Build the customized D version. 54 | /// The result will be in resultDir. 55 | void runBuild(string spec, DManager.SubmoduleState submoduleState, bool asNeeded) 56 | { 57 | auto buildInfoPath = buildPath(resultDir, buildInfoFileName); 58 | auto buildInfo = BuildInfo(diggerVersion, spec, d.config.build, submoduleState); 59 | if (asNeeded && buildInfoPath.exists && buildInfoPath.readText.jsonParse!BuildInfo == buildInfo) 60 | { 61 | log("Reusing existing version in " ~ resultDir); 62 | return; 63 | } 64 | d.build(submoduleState); 65 | prepareResult(); 66 | std.file.write(buildInfoPath, buildInfo.toJson()); 67 | } 68 | 69 | /// Perform an incremental build, i.e. don't clean or fetch anything from remote repos 70 | void incrementalBuild() 71 | { 72 | d.rebuild(); 73 | prepareResult(); 74 | } 75 | 76 | /// Run tests. 77 | void runTests() 78 | { 79 | d.test(); 80 | } 81 | 82 | DManager.SubmoduleState parseSpec(string spec) 83 | { 84 | auto parts = spec.split("+"); 85 | parts = parts.map!strip().array(); 86 | if (parts.empty) 87 | parts = [null]; 88 | auto rev = parseRev(parts.shift()); 89 | 90 | auto state = d.begin(rev); 91 | 92 | foreach (part; parts) 93 | { 94 | bool revert = part.skipOver("-"); 95 | 96 | void apply(string component, string[2] branch, DManager.MergeMode mode) 97 | { 98 | if (revert) 99 | d.revert(state, component, branch, mode); 100 | else 101 | d.merge(state, component, branch, mode); 102 | } 103 | 104 | if (part.matchCaptures(re!`^(\w[\w\-\.]*)#(\d+)$`, 105 | (string component, int pull) 106 | { 107 | apply(component, d.getPull(component, pull), DManager.MergeMode.cherryPick); 108 | })) 109 | continue; 110 | 111 | if (part.matchCaptures(re!`^(?:([a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])/)?(\w[\w\-\.]*)/(?:(\w[\w\-]*)\.\.)?(\w[\w\-]*)$`, 112 | (string user, string component, string base, string tip) 113 | { 114 | // Some "do what I mean" logic here: if the user 115 | // specified a range, or a single commit, cherry-pick; 116 | // otherwise (just a branch name), do a git merge 117 | auto branch = d.getBranch(component, user, base, tip); 118 | auto mode = branch[0] ? DManager.MergeMode.cherryPick : DManager.MergeMode.merge; 119 | apply(component, branch, mode); 120 | })) 121 | continue; 122 | 123 | throw new Exception("Don't know how to apply customization: " ~ spec); 124 | } 125 | 126 | return state; 127 | } 128 | 129 | /// Build D according to the given spec string 130 | /// (e.g. master+dmd#123). 131 | void buildCustom(string spec, bool asNeeded = false) 132 | { 133 | log("Building spec: " ~ spec); 134 | auto submoduleState = parseSpec(spec); 135 | runBuild(spec, submoduleState, asNeeded); 136 | } 137 | 138 | void checkout(string spec) 139 | { 140 | log("Checking out: " ~ spec); 141 | auto submoduleState = parseSpec(spec); 142 | d.checkout(submoduleState); 143 | log("Done."); 144 | } 145 | 146 | /// Build all D versions (for the purpose of caching them). 147 | /// Build order is in steps of decreasing powers of two. 148 | void buildAll(string spec) 149 | { 150 | d.needUpdate(); 151 | auto commits = d.getLog("refs/remotes/origin/" ~ spec); 152 | commits.reverse(); // oldest first 153 | 154 | for (int step = 1 << 30; step; step >>= 1) 155 | { 156 | if (step >= commits.length) 157 | continue; 158 | 159 | log("Building all revisions with step %d (%d/%d revisions)".format(step, commits.length/step, commits.length)); 160 | 161 | for (int n = step; n < commits.length; n += step) 162 | try 163 | { 164 | auto state = d.begin(commits[n].hash); 165 | if (!d.isCached(state)) 166 | { 167 | log("Building revision %d/%d".format(n/step, commits.length/step)); 168 | d.build(state); 169 | } 170 | } 171 | catch (Exception e) 172 | log(e.toString()); 173 | } 174 | } 175 | -------------------------------------------------------------------------------- /digger.ini.sample: -------------------------------------------------------------------------------- 1 | # Copy this file to digger.ini, and make the appropriate 2 | # changes below. 3 | 4 | # Digger looks for digger.ini in these locations, in order: 5 | # - current directory 6 | # - EXE directory 7 | # - source directory 8 | # - On POSIX: 9 | # - $XDG_CONFIG_HOME/digger/ (default ~/.config/digger/) 10 | # - $XDG_CONFIG_DIRS/digger/ (default /etc/xdg/digger/) 11 | # - On Windows: 12 | # - %APPDATA%\Digger\ 13 | # (default C:\Users\\AppData\Roaming\Digger) 14 | # 15 | # Additionally, for compatibility with previous versions, 16 | # the following locations are searched: 17 | # - ~/.digger/digger.ini 18 | # - On POSIX: /etc/digger.ini. 19 | 20 | # Syntax: 21 | # The section names define an implicit prefix, thus ... 22 | # [a.b] 23 | # c.d=e 24 | # ... is equivalent to ... 25 | # a.b.c.d=e 26 | 27 | # These settings can also be set from the command line using 28 | # the -c option, and also overridden in a bisect.ini file, 29 | # e.g.: -c build.components.dmd.debugDMD=true 30 | 31 | # Local configuration. 32 | 33 | [local] 34 | 35 | # Working directory. 36 | # This directory will contain all of Digger's working files: 37 | # the D repositories, any build prerequisites obtained 38 | # automatically, the current build output, and the cache, if 39 | # enabled. 40 | # Please specify an absolute path. The default is to use the 41 | # current directory. 42 | 43 | #workDir = C:\Temp\Digger 44 | 45 | # How many jobs to run makefiles in. 46 | # Gets passed to GNU make as the -j parameter (not supported by 47 | # DigitalMars make on Windows). Specify "auto" to use the 48 | # CPU core count, or "unlimited" for no limit. 49 | # Equivalent to the --jobs command-line option. 50 | 51 | #makeJobs = auto 52 | 53 | # Don't go online to fetch the latest revisions from GitHub. 54 | # Equivalent to the --offline command-line switch. 55 | 56 | #offline = false 57 | 58 | # Build cache. 59 | # To speed up successive runs, Digger can save the results of 60 | # each commit's build. The downside is that this uses up disk 61 | # space. The following cache engines are available: 62 | # - none No persistent cache. 63 | # - directory Store built files in a directory tree. 64 | # Saves some disk space by hard-linking identical 65 | # files. 66 | # - git Use a git repository (and git's deduplication / 67 | # compression mechanisms). Uses much less disk 68 | # space than "directory", but is a little slower. 69 | # You can periodically run "digger compact" to optimize disk 70 | # space used by the cache. 71 | 72 | #cache = git 73 | 74 | # Default build options. 75 | 76 | [build] 77 | 78 | # Enable or disable D components to build. 79 | # For example, rdmd is rarely needed, so we can disable it here. 80 | # Additional components not enabled by default can also be added. 81 | # Equivalent to the --with and --without "digger build" options. 82 | # Run `digger build --help` to see a list of available components. 83 | 84 | #components.enable.rdmd = false 85 | 86 | # Target model. 87 | # Whether to target 32 or 64 bits when building D components. 88 | # On Windows, 32mscoff is also an option. 89 | # The default is to use the system model on POSIX, and 32-bit 90 | # on Windows. Equivalent to the `digger build --model` option. 91 | # Multiple models can be specified simultaneously by 92 | # comma-separating them. 93 | 94 | #components.common.model = 32,64 95 | 96 | # Additional make parameters. 97 | # Equivalent to the --make-args "digger build" option. 98 | 99 | #components.common.makeArgs = ["HOST_CC=g++48"] 100 | 101 | # Whether to build a debug compiler. 102 | # Debug compilers are built quicker, but compile D code slower. 103 | 104 | #components.dmd.debugDMD = false 105 | 106 | # Whether to build a release compiler. 107 | # Enables optimizations. Mutually exclusive with the above. 108 | 109 | #components.dmd.releaseDMD = false 110 | 111 | # Model for building DMD itself (on Windows). 112 | # Can be used to build a 64-bit DMD, to avoid 4GB limit. 113 | # Currently only implemented on Windows, for DMD 2.072 or later. 114 | # Can also be set to 32mscoff. 115 | 116 | #components.dmd.dmdModel = 64 117 | 118 | # How to obtain a D compiler to build the parts of DMD written in D. 119 | 120 | # Digger can either download a pre-built D version (default), 121 | # or build one from source. 122 | # Equivalent to the `digger build --bootstrap` option. 123 | 124 | #components.dmd.bootstrap.fromSource = false 125 | 126 | # Which D version to use when building the D compiler. 127 | # The default is to select one automatically. 128 | # This can be a version name, such as "v2.070.2", 129 | # but also an arbitrary version specification 130 | # (only when bootstrapping from source), 131 | # as when using "digger build". 132 | 133 | #components.dmd.bootstrap.ver = v2.067.1 134 | 135 | # Build configuration for the compiler used for bootstrapping. 136 | # If no options are set, the default settings are used, 137 | # so specify any pertinent build settings that apply to both 138 | # the host (bootstrapping) and built compiler here as well. 139 | # Used when fromSource is true. 140 | 141 | #components.dmd.bootstrap.build.option = value 142 | 143 | # Note that it is possible to bootstrap the bootstrapping 144 | # compiler, up to an arbitrary depth. For example, to build 145 | # 2.067.1 from source, then use it to build master, then 146 | # build master again using the compiler just built from master: 147 | 148 | # $ digger \ 149 | # -c "build.components.dmd.bootstrap.build.components.dmd.bootstrap.fromSource = true" \ 150 | # -c "build.components.dmd.bootstrap.build.components.dmd.bootstrap.ver = v2.067.1" \ 151 | # -c "build.components.dmd.bootstrap.fromSource = true" \ 152 | # -c "build.components.dmd.bootstrap.ver = master" \ 153 | # build master 154 | 155 | # Build/test environment. 156 | # By default, Digger completely clears the environment and 157 | # builds a new one from scratch, to avoid potential sources 158 | # of contamination that can affect the D builds or test results. 159 | [build.environment] 160 | 161 | # You can use %VAR% syntax to refer to the previous value of a 162 | # variable, or if there wasn't one, to the value from the 163 | # original host environment (before it was cleared and rebuilt). 164 | 165 | # Examples: 166 | 167 | # Add something to PATH 168 | #PATH=%PATH%;C:\Tools 169 | 170 | # Import PATHEXT from the original environment 171 | #PATHEXT=%PATHEXT% 172 | 173 | # There are some additional lesser-used options not listed here, 174 | # see ae.sys.d.manager.DManager.Config for details. 175 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | Digger Changelog 2 | ================ 3 | 4 | Digger v3.0 (2020-04-21) 5 | ------------------------ 6 | 7 | * Major internal changes for improved reliability 8 | * Cache version bumped to 3 9 | * Updated, backwards-incompatible .ini settings 10 | * All settings are now equally available from `digger.ini`, `bisect.ini` 11 | and command line 12 | * Search for the configuration file according to the XDG Base Directory 13 | Specification 14 | * Add `-c` options to specify arbitrary `digger.ini` and `bisect.ini` 15 | settings on the command line 16 | * Specifying a `bisect.ini` for the `bisect` command is now optional, 17 | and can be entirely substituted with `-c` options. 18 | * Add `digger checkout` command, which simply checks out a given D revision 19 | (`master` by default) 20 | * Add `digger test` command, to run tests for working tree state 21 | * Add `digger run` command, which runs arbitrary commands in the 22 | context of some D version 23 | * Add ability to revert a branch or pull request. 24 | The syntax is to prefix the branch or PR with a `-` (minus sign). 25 | Example: `digger build "master + -phobos#1234"` 26 | * Add ability to specify commit SHA1 instead of a branch or PR number. 27 | Example: `digger build "master + -dmd/0123456789abcdef0123456789abcdef01234567"` 28 | * Add ability to specify a tag instead of a timestamp to use that tag's time 29 | Example: `digger build "master @ v2.085.0"` 30 | * Add `tools`, `extras` and `curl` components 31 | * Add `32mscoff` model support for Windows 32 | * Add `--jobs` option for controlling the GNU make `-j` parameter 33 | * Add `components.dmd.releaseDMD` build option to complement `debugDMD` 34 | * Add `components.dmd.dmdModel` option, which allows building a 64-bit 35 | `dmd.exe` on Windows (also supports `32mscoff`). 36 | * Improve bootstrapping up to arbitrary depths 37 | * Refuse to clobber working tree changes not done by Digger 38 | * Verify integrity of all downloaded files 39 | * Only download/install Visual Studio components as-needed 40 | * Prevent git from loading user/system configuration 41 | * Add `app.gitOptions` setting, which allows e.g. specifying an 42 | alternate way to clone git:// URLs 43 | * Add `dub.sdl` 44 | * Add `DIGGER_REVISION` environment variable during bisection 45 | * Add `extraSpec` bisect option, which allows specifying additional 46 | patches to apply during bisection 47 | * Add test suite 48 | * Enable continuous integration on Travis and AppVeyor 49 | * Remove `digger-web` 50 | * Various fixes 51 | 52 | Digger v2.4 (2015-10-05) 53 | ------------------------ 54 | 55 | * Fetch tags explicitly when updating 56 | (fixes some "unknown /ambiguous revision" errors) 57 | * Prepend result `bin` directory to `PATH` 58 | (fixes behavior when a `dmd` binary was installed in `/usr/bin`) 59 | * Add support for the `debugDMD` build option on POSIX 60 | * Fix incorrect repository tree order when using `git` cache engine 61 | * Fix `rebuild` ignoring build options on the command-line 62 | * Automatically install KindleGen locally when building website 63 | * Update OptLink installer 64 | * Download platform-specific DMD release packages 65 | (contributed by Martin Nowak) 66 | 67 | Digger v2.3 (2015-06-14) 68 | ------------------------ 69 | 70 | * Add `bisectBuild` bisect config option 71 | * Add `--with` and `--without` switches to control D components to build 72 | * Add `website` component for building dlang.org (POSIX-only) 73 | * Work around `appender` memory corruption bug with `git` cache engine 74 | * Various fixes 75 | 76 | Digger v2.2 (2015-06-05) 77 | ------------------------ 78 | 79 | * Fix `digger install` to work with `object.d` 80 | * Improve resilience of `digger install` 81 | * Add `--bootstrap` switch to build compiler entirely from C++ source 82 | * Replace usage of `git bisect run` with internal implementation 83 | * Bisection now prefers cached builds when choosing a commit to test 84 | * Allow customizing the set of components to build during bisection 85 | * Use git plumbing in git cache driver for concurrency and better performance 86 | * Don't cache build failures if the error is likely temporary 87 | 88 | Digger v2.1 (2015-05-03) 89 | ------------------------ 90 | 91 | * Add [license](LICENSE.md) 92 | * Add `git` cache engine 93 | * Add `cache` action and subcommands 94 | * Fix starting `digger-web` in OS X 95 | (auto-correct working directory) 96 | 97 | Digger v2.0 (2015-04-26) 98 | ------------------------ 99 | 100 | * `idgen.d` update (DMD now requires DMD to build) 101 | * Full core overhaul, for improved performance, granularity and extensibility. 102 | A fresh install is recommended. 103 | 104 | Digger v1.1 (2015-03-04) 105 | ------------------------ 106 | 107 | * Add `rebuild` action, for incremental rebuilds 108 | (thanks to Sergei Nosov) 109 | * Add `install` and `uninstall` actions 110 | * Add `--help` text 111 | * Add `--make-args` option 112 | * Add `--model` option to replace the `--64` switch 113 | * Add `--host` and `--port` to `digger-web` 114 | * Various smaller improvements 115 | 116 | Digger v1.0 (2014-09-18) 117 | ------------------------ 118 | 119 | * On Windows, Digger may now download and locally install (unpack) required 120 | software, as needed: 121 | - Git 122 | - A number of Visual Studio 2013 Express and Windows SDK components (for 123 | 64-bit builds) 124 | - 7-Zip and WiX (necessary for unpacking Visual Studio Express components) 125 | * Various smaller improvements 126 | 127 | Digger v0.3 (2014-05-22) [DConf edition] 128 | ---------------------------------------- 129 | 130 | * Allow merging arbitrary GitHub forks 131 | * Add `--offline`, which suppresses updating the D repositories. 132 | * Move digger-web tasks to digger, thus removing D building logic from 133 | digger-web binary 134 | * Improve revision parsing, allowing e.g. `digger build 2.065 @ 3 weeks ago` 135 | * Rename `digger-web` directory to `web`, to avoid conflict with POSIX binary 136 | of `digger-web.d` 137 | * Fix web UI behavior when refreshing 138 | * Fix exit status code propagation 139 | * Various smaller improvements 140 | 141 | Digger v0.2 (2014-04-01) [April Fools' edition] 142 | ----------------------------------------------- 143 | 144 | * Add `digger-web` 145 | * Fix parsing Environment configuration section 146 | * Various smaller improvements 147 | 148 | Digger v0.1 (2014-02-17) [Initial release] 149 | ------------------------------------------ 150 | 151 | * Initial announcement 152 | 153 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Digger [![Build Status](https://travis-ci.org/CyberShadow/Digger.svg?branch=master)](https://travis-ci.org/CyberShadow/Digger) [![AppVeyor](https://ci.appveyor.com/api/projects/status/tm98i6iw931ma3yg/branch/master?svg=true)](https://ci.appveyor.com/project/CyberShadow/digger) 2 | 3 | Digger can: 4 | 5 | - build and test D from [git](https://github.com/dlang) 6 | - build older versions of D 7 | - build D plus forks and pending pull requests 8 | - bisect D's history to find where regressions are introduced (or bugs fixed) 9 | 10 | ### Requirements 11 | 12 | On POSIX, Digger needs git, g++, binutils and make. gcc-multilib and g++-multilib or equivalent are required for targeting x86 on x86_64 systems. 13 | 14 | On Windows, Digger will download and unpack everything it needs (Git, DMC, DMD, 7-Zip, WiX, VS2013 and Windows SDK components). 15 | 16 | ### Get Digger 17 | 18 | The easiest way to obtain and run Digger is through Dub. As Dub is included with DMD, simply run: 19 | 20 | $ dub fetch digger 21 | $ dub run digger -- ARGS... 22 | 23 | where `ARGS...` are the Digger command and arguments (see below). 24 | 25 | ### Command-line usage 26 | 27 | (The command line examples below assume that Digger is installed in a location under `PATH` - 28 | replace `digger ...` with `./digger ...` or `dub run digger -- ...` as appropriate.) 29 | 30 | ##### Building D 31 | 32 | # build latest master branch commit 33 | $ digger build 34 | 35 | # build a specific D version 36 | $ digger build v2.064.2 37 | 38 | # build for x86-64 39 | $ digger build --model=64 v2.064.2 40 | 41 | # build commit from a point in time 42 | $ digger build "@ 3 weeks ago" 43 | 44 | # build latest 2.065 (release) branch commit 45 | $ digger build 2.065 46 | 47 | # build specified branch from a point in time 48 | $ digger build "2.065 @ 3 weeks ago" 49 | 50 | # build with added pull request 51 | $ digger build "master + dmd#123" 52 | 53 | # build with added GitHub fork branch 54 | $ digger build "master + Username/dmd/awesome-feature" 55 | 56 | # build with reverted commit 57 | $ digger build "master + -dmd/0123456789abcdef0123456789abcdef01234567" 58 | 59 | # build with reverted pull request 60 | $ digger build "master + -dmd#123" 61 | 62 | ##### Building D programs 63 | 64 | # Run the last built DMD version 65 | $ digger run - -- dmd --help 66 | 67 | # Build and run latest DMD master 68 | $ digger run master -- dmd --help 69 | 70 | # Build latest DMD master, and then build and run a D program using it 71 | $ digger run master -- dmd -i -run program.d 72 | 73 | ##### Hacking on D 74 | 75 | # check out git master (or some other version) 76 | $ digger checkout 77 | 78 | # build / incrementally rebuild current checkout 79 | $ digger rebuild 80 | 81 | # run tests 82 | $ digger test 83 | 84 | Run `digger` with no arguments for detailed usage help. 85 | 86 | ##### Installing 87 | 88 | Digger does not build all D components - only those that change frequently and depend on one another. 89 | You can get a full package by upgrading an installation of a stable DMD release with Digger's build result: 90 | 91 | # upgrade the DMD in your PATH with Digger's result 92 | $ digger install 93 | 94 | You can undo this at any time by running: 95 | 96 | $ digger uninstall 97 | 98 | Successive installs will not clobber the backups created by the first `digger install` invocation, 99 | so `digger uninstall` will revert to the state from before you first ran `digger install`. 100 | 101 | You can also simultaneously install 32-bit and 64-bit versions of Phobos by first building and installing a 32-bit DMD, 102 | then a 64-bit DMD (`--model=64`). `digger uninstall` will revert both actions. 103 | 104 | To upgrade a system install of DMD on POSIX, simply run `digger install` as root, e.g. `sudo digger install`. 105 | 106 | `digger install` should be compatible with [DVM](https://github.com/jacob-carlborg/dvm) or any other DMD installation. 107 | 108 | Installation and uninstallation are designed with safety in mind. 109 | When installing, Digger will provide detailed information and ask for confirmation before making any changes 110 | (you can use the `--dry-run` / `--yes` switches to suppress the prompt). 111 | Uninstallation will refuse to remove files if they were modified since they were installed, 112 | to prevent accidentally clobbering user work (you can use `--force` to override this). 113 | 114 | ##### Bisecting 115 | 116 | To bisect D's history to find which pull request introduced a bug, first copy `bisect.ini.sample` to `bisect.ini`, adjust as instructed by the comments, then run: 117 | 118 | $ digger bisect path/to/bisect.ini 119 | 120 | If Digger ends up with a master/stable merge as the bisection result, switch the branches on the starting points accordingly, e.g.: 121 | 122 | - If you specified `good=v2.080.0` and `bad=v2.081.0`, try `good=master@v2.080.0` and `bad=master@v2.081.0` 123 | - If you specified `good=@2018-01-01` and `bad=@2019-01-01`, try `good=stable@2018-01-01` and `bad=stable@2019-01-01` 124 | - Note that the master/stable branch-offs/merges do not happen at the same time as when releases are tagged, 125 | so you may need to increase the bisection range accordingly. See [DIP75](https://wiki.dlang.org/DIP75) for details. 126 | 127 | ### Configuration 128 | 129 | You can optionally configure a few settings using a configuration file. 130 | To do so, copy `digger.ini.sample` to `digger.ini` and adjust as instructed by the comments. 131 | 132 | ### Building 133 | 134 | $ git clone --recursive https://github.com/CyberShadow/Digger 135 | $ cd Digger 136 | $ rdmd --build-only digger 137 | 138 | * If you get a link error, you may need to add `-allinst` or `-debug` due to [a DMD bug](https://github.com/CyberShadow/Digger/issues/37). 139 | 140 | * A [dub](https://code.dlang.org/) definition is also included. 141 | 142 | * On Windows, you may see: 143 | 144 | Warning 2: File Not Found version.lib 145 | 146 | This is a benign warning. 147 | 148 | ### Hacking 149 | 150 | ##### Backend 151 | 152 | The code which builds D and manages the git repository is located in the [ae library](https://github.com/CyberShadow/ae) 153 | (`ae.sys.d` package), so as to be reusable. 154 | 155 | Currently, the bulk of the code is in [`ae.sys.d.manager`](https://github.com/CyberShadow/ae/blob/master/sys/d/manager.d). 156 | 157 | `ae.sys.d.manager` clones [a meta-repository on BitBucket](https://bitbucket.org/cybershadow/d), which contains the major D components as submodules. 158 | The meta-repository is created and maintained by another program, [D-dot-git](https://github.com/CyberShadow/D-dot-git). 159 | 160 | The build requirements are fulfilled by the [`ae.sys.install`](https://github.com/CyberShadow/ae/tree/master/sys/install) package. 161 | 162 | ##### Frontend 163 | 164 | Digger is the frontend to the above library code, implementing configuration, bisecting, etc. 165 | 166 | Module list is as follows: 167 | 168 | - `config` - configuration 169 | - `repo` - customized D repository management, revision parsing 170 | - `bisect` - history bisection 171 | - `custom` - custom build management 172 | - `install` - installation 173 | - `digger` - entry point and command-line UI 174 | - `common` - shared helpers 175 | 176 | ### Remarks 177 | 178 | ##### Wine 179 | 180 | Digger should work fine under Wine. For 64-bit builds, you must first run `winetricks vcrun2013`. 181 | Digger cannot do this automatically as this must be done from outside Wine. 182 | -------------------------------------------------------------------------------- /test/run-tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | # Globals 5 | 6 | uname="$(uname)" 7 | 8 | # Setup / cleanup 9 | 10 | function init() { 11 | cd "$(dirname "$0")" 12 | 13 | echo "local.workDir = work/" > ./digger.ini 14 | echo "local.cache = git" >> ./digger.ini 15 | echo "local.makeJobs = auto" >> ./digger.ini 16 | 17 | if [[ -n "${GITHUB_API_TOKEN:-}" ]] 18 | then 19 | echo "local.githubToken = $GITHUB_API_TOKEN" >> ./digger.ini 20 | fi 21 | 22 | rm -rf digger work 23 | ( shopt -s nullglob; rm -f ./*.lst ) 24 | 25 | if [[ "$uname" != *_NT-* ]] 26 | then 27 | # Apply -no-pie workaround to programs built here, too 28 | PATH=$PWD/work/bin:$PATH 29 | fi 30 | } 31 | 32 | # Common functions 33 | 34 | function xfail() { 35 | if "$@" ; then false ; fi 36 | } 37 | 38 | function build() { 39 | local dflags=(-cov -debug -g '-version=test') 40 | rdmd --build-only "${dflags[@]}" "$@" -of./digger ../digger.d 41 | } 42 | 43 | function clean() { 44 | pushd work/repo/ 45 | git submodule foreach git reset --hard 46 | git submodule foreach git clean -fdx 47 | popd 48 | } 49 | 50 | function digger() { 51 | if [[ ! -x ./digger ]] ; then build ; fi 52 | ./digger --config-file ./digger.ini "$@" 53 | } 54 | 55 | # Run unittests 56 | 57 | function test_unit() { 58 | rm -f ./digger 59 | build -unittest 60 | ./digger build --help 61 | rm ./digger 62 | } 63 | 64 | # Simple build 65 | 66 | function test_build() { 67 | digger build "master @ 2016-01-01 00:00:00" 68 | 69 | work/result/bin/dmd -run issue15914.d 70 | } 71 | 72 | # Run tests 73 | 74 | function test_testsuite() { 75 | digger build "master @ 2018-04-01 00:00:00 + tools#346 + tools#347" 76 | 77 | clean # Clean everything to test correct test dependencies 78 | 79 | local test_args=('--with=phobos' '--with=tools') 80 | if [[ "$uname" == "Darwin" ]] 81 | then 82 | # TODO, rdmd bug: https://travis-ci.org/CyberShadow/Digger/jobs/124429436 83 | test_args+=('--without=rdmd') 84 | fi 85 | if [[ "${APPVEYOR:-}" == "True" ]] 86 | then 87 | # MSYS downloads fail on AppVeyor (bandwidth quota exceeded?) 88 | test_args+=('--without=dmd') 89 | # TODO, Druntime tests segfault on AppVeyor 90 | test_args+=('--without=druntime') 91 | fi 92 | 93 | if [[ "$uname" == *_NT-* ]] 94 | then 95 | # Test 64-bit on Windows too 96 | digger test "${test_args[@]}" --model=64 97 | clean # Needed to rebuild zlib for correct model 98 | # Build 64-bit first so we leave behind 32-bit C++ object files, 99 | # so that the rebuild action below succeeds. 100 | fi 101 | 102 | digger test "${test_args[@]}" 103 | } 104 | 105 | # Caching 106 | 107 | function test_cache() { 108 | digger build "master @ 2016-01-01 00:00:00" 109 | 110 | digger --offline build "master @ 2016-01-01 00:00:00" 2>&1 | tr -d '\r' | tee digger.log 111 | xfail grep --quiet --fixed-strings --line-regexp 'digger: Cache miss.' digger.log 112 | grep --quiet --fixed-strings --line-regexp 'digger: Cache hit!' digger.log 113 | } 114 | 115 | # Caching unbuildable versions 116 | 117 | function test_cache_error() { 118 | xfail digger build "master @ 2009-07-01 00:00:00" 119 | 120 | xfail digger --offline build "master @ 2009-07-01 00:00:00" 2>&1 | tr -d '\r' | tee digger.log 121 | xfail grep --quiet --fixed-strings --line-regexp 'digger: Cache miss.' digger.log 122 | grep --quiet --fixed-strings --line-regexp 'digger: Cache hit!' digger.log 123 | grep --quiet --fixed-strings 'was cached as unbuildable' digger.log 124 | } 125 | 126 | # Rebuild 127 | 128 | function test_rebuild() { 129 | digger checkout "master @ 2016-01-01 00:00:00" 130 | digger build "master @ 2016-01-01 00:00:00" 131 | 132 | # No changes 133 | 134 | digger rebuild 135 | work/result/bin/dmd -run issue15914.d 136 | 137 | # With changes 138 | 139 | pushd work/repo/phobos/ 140 | git cherry-pick --no-commit ad226e92d5f092df233b90fd3fdedb8b71d728eb 141 | popd 142 | 143 | digger rebuild 144 | xfail work/result/bin/dmd -run issue15914.d 145 | 146 | rm work/repo/.git/modules/phobos/ae-sys-d-worktree.json 147 | } 148 | 149 | # Working tree state 150 | 151 | function test_worktree() { 152 | digger checkout "master @ 2016-01-01 00:00:00" 153 | 154 | pushd work/repo/phobos/ 155 | git cherry-pick --no-commit ad226e92d5f092df233b90fd3fdedb8b71d728eb 156 | popd 157 | 158 | xfail digger build "master @ 2016-04-01 00:00:00" # Worktree is dirty - should fail 159 | rm work/repo/.git/modules/phobos/ae-sys-d-worktree.json 160 | digger build "master @ 2016-04-01 00:00:00" # Should work now 161 | xfail work/result/bin/dmd -run issue15914.d 162 | } 163 | 164 | # Merging 165 | 166 | function test_merge() { 167 | digger build "master @ 2016-01-01 00:00:00 + phobos#3859" 168 | xfail work/result/bin/dmd -run issue15914.d 169 | 170 | # Test cache 171 | 172 | digger --offline build "master @ 2016-01-01 00:00:00 + phobos#3859" 2>&1 | tr -d '\r' | tee digger.log 173 | grep --quiet --fixed-strings --line-regexp 'digger: Merging phobos commits 791659460d83b4d92c232c6a87a39276a842388c..ad226e92d5f092df233b90fd3fdedb8b71d728eb' digger.log 174 | xfail grep --quiet --fixed-strings --line-regexp 'digger: Cache miss.' digger.log 175 | grep --quiet --fixed-strings --line-regexp 'digger: Cache hit!' digger.log 176 | } 177 | 178 | # Reverting 179 | 180 | function test_revert() { 181 | digger --offline build "master @ 2016-04-01 00:00:00 + -phobos#3859" 182 | work/result/bin/dmd -run issue15914.d 183 | } 184 | 185 | # Bisecting 186 | 187 | function test_bisect() { 188 | cat > bisect.ini <&1 | tr -d '\r' | tee digger.log 196 | 197 | if [[ "$uname" == *_NT-* ]] 198 | then 199 | # Digger outputs \r\n newlines in its log output on Windows, filter those out before diffing 200 | diff <(tail -n 19 digger.log | sed 's/\r//' | grep -v '^index ') issue15914-bisect.log 201 | else 202 | # OS X doesn't support the \r escape 203 | diff <(tail -n 19 digger.log | grep -v '^index ') issue15914-bisect.log 204 | fi 205 | } 206 | 207 | # Test building model combinations 208 | 209 | function test_model() { 210 | local models 211 | if [[ "$uname" == *_NT-* ]] 212 | then 213 | models=(32 64 32mscoff) 214 | else 215 | models=(32 64) 216 | fi 217 | 218 | for model in "${models[@]}" 219 | do 220 | for dmdmodel in "${models[@]}" 221 | do 222 | digger -c build.components.dmd.dmdModel="$dmdmodel" build --model="$model" "master@2016-04-01+dmd#5694" 223 | work/result/bin/dmd "-m${model}" -run hello.d 224 | done 225 | done 226 | } 227 | 228 | # Build test - stable @ 2015-09-01 00:00:00 (Phobos can't find Druntime .a / .so by default) 229 | 230 | function test_2015_09_01() { 231 | if [[ ! "$uname" == *_NT-* ]] 232 | then 233 | digger build "stable @ 2015-09-01 00:00:00" 234 | fi 235 | } 236 | 237 | # Build test - master @ 2019-10-10 00:00:00 (build.d trouble on Windows) 238 | 239 | function test_2019_10_10() { 240 | digger build "master @ 2019-10-10 00:00:00" 241 | } 242 | 243 | # Build test - v2.094.2 (Druntime mak/copyimports.d breakage on Windows) 244 | 245 | function test_2_094_2() { 246 | digger build "v2.094.2" 247 | } 248 | 249 | # The test runner 250 | 251 | function run_tests() { 252 | init 253 | 254 | local t 255 | for t in "$@" 256 | do 257 | echo "=== Running test $t ===" 258 | ( set -x ; "test_$t" ) 259 | echo "=== Test $t OK! ===" 260 | done 261 | 262 | echo -e "==================================================================\nAll tests OK!" 263 | } 264 | 265 | # Main function 266 | 267 | function main() { 268 | local all_tests=( 269 | unit 270 | build 271 | testsuite 272 | cache 273 | cache_error 274 | rebuild 275 | worktree 276 | merge 277 | revert 278 | bisect 279 | model 280 | 2015_09_01 281 | # 2019_10_10 282 | # 2_094_2 283 | ) 284 | 285 | if [[ $# -eq 0 ]] 286 | then 287 | run_tests "${all_tests[@]}" 288 | else 289 | run_tests "$@" 290 | fi 291 | } 292 | 293 | main "$@" 294 | -------------------------------------------------------------------------------- /digger.d: -------------------------------------------------------------------------------- 1 | module digger; 2 | 3 | import std.array; 4 | import std.exception; 5 | import std.file : thisExePath, exists; 6 | import std.path; 7 | import std.process; 8 | import std.stdio; 9 | import std.typetuple; 10 | 11 | static if(!is(typeof({import ae.utils.text;}))) static assert(false, "ae library not found, did you clone with --recursive?"); else: 12 | 13 | version (Windows) 14 | static import ae.sys.net.wininet; 15 | else 16 | static import ae.sys.net.curl; 17 | 18 | import ae.sys.d.manager : DManager; 19 | import ae.utils.funopt; 20 | import ae.utils.main; 21 | import ae.utils.meta : structFun; 22 | import ae.utils.text : eatLine; 23 | 24 | import bisect; 25 | import common; 26 | import config; 27 | import custom; 28 | import install; 29 | import repo; 30 | 31 | // http://d.puremagic.com/issues/show_bug.cgi?id=7016 32 | version(Windows) static import ae.sys.windows; 33 | 34 | alias BuildOptions(string action, string pastAction, bool showBuildActions = true) = TypeTuple!( 35 | Switch!(hiddenOption, 0, "64"), 36 | Option!(string, showBuildActions ? "Select models (32, 64, or, on Windows, 32mscoff). You can specify multiple models by comma-separating them.\nOn this system, the default is " ~ DManager.Config.Build.components.common.defaultModel ~ " [build.components.common.models]" : hiddenOption, null, 0, "model"), 37 | Option!(string[], "Do not " ~ action ~ " a component (that would otherwise be " ~ pastAction ~ " by default). List of default components: " ~ DManager.defaultComponents.join(", ") ~ " [build.components.enable.COMPONENT=false]", "COMPONENT", 0, "without"), 38 | Option!(string[], "Specify an additional D component to " ~ action ~ ". List of available additional components: " ~ DManager.additionalComponents.join(", ") ~ " [build.components.enable.COMPONENT=true]", "COMPONENT", 0, "with"), 39 | Option!(string[], showBuildActions ? `Additional make parameters, e.g. "HOST_CC=g++48" [build.components.common.makeArgs]` : hiddenOption, "ARG", 0, "makeArgs"), 40 | Switch!(showBuildActions ? "Bootstrap the compiler (build from C++ source code) instead of downloading a pre-built binary package [build.components.dmd.bootstrap.fromSource]" : hiddenOption, 0, "bootstrap"), 41 | Switch!(hiddenOption, 0, "use-vc"), 42 | Switch!(hiddenOption, 0, "clobber-local-changes"), 43 | ); 44 | 45 | enum specDescription = "D ref (branch / tag / point in time) to build, plus any additional forks or pull requests. Example:\n" ~ 46 | "\"master @ 3 weeks ago + dmd#123 + You/dmd/awesome-feature\""; 47 | 48 | void parseBuildOptions(T...)(T options) // T == BuildOptions!action 49 | { 50 | if (options[0]) 51 | d.config.build.components.common.models = ["64"]; 52 | if (options[1]) 53 | d.config.build.components.common.models = options[1].value.split(","); 54 | foreach (componentName; options[2]) 55 | d.config.build.components.enable[componentName] = false; 56 | foreach (componentName; options[3]) 57 | d.config.build.components.enable[componentName] = true; 58 | d.config.build.components.common.makeArgs ~= options[4]; 59 | d.config.build.components.dmd.bootstrap.fromSource |= options[5]; 60 | d.config.build.components.dmd.useVC |= options[6]; 61 | d.verifyWorkTree = !options[7]; 62 | static assert(options.length == 8); 63 | } 64 | 65 | struct Digger 66 | { 67 | static: 68 | @(`Build D from source code`) 69 | int build(BuildOptions!("build", "built") options, Parameter!(string, specDescription) spec = "master") 70 | { 71 | parseBuildOptions(options); 72 | buildCustom(spec); 73 | return 0; 74 | } 75 | 76 | @(`Incrementally rebuild the current D checkout`) 77 | int rebuild(BuildOptions!("rebuild", "rebuilt") options) 78 | { 79 | parseBuildOptions(options); 80 | incrementalBuild(); 81 | return 0; 82 | } 83 | 84 | @(`Run tests for enabled components`) 85 | int test(BuildOptions!("test", "tested") options) 86 | { 87 | parseBuildOptions(options); 88 | runTests(); 89 | return 0; 90 | } 91 | 92 | @(`Check out D source code from git`) 93 | int checkout(BuildOptions!("check out", "checked out", false) options, Parameter!(string, specDescription) spec = "master") 94 | { 95 | parseBuildOptions(options); 96 | .checkout(spec); 97 | return 0; 98 | } 99 | 100 | @(`Run a command using a D version`) 101 | int run( 102 | BuildOptions!("build", "built") options, 103 | Parameter!(string, specDescription ~ "\nSpecify \"-\" to use the previously-built version.") spec, 104 | Parameter!(string[], "Command to run and its arguments (use -- to pass switches)") command) 105 | { 106 | if (spec == "-") 107 | enforce(options == typeof(options).init, "Can't specify build options when using the last built version."); 108 | else 109 | { 110 | parseBuildOptions(options); 111 | buildCustom(spec, /*asNeeded*/true); 112 | } 113 | 114 | auto binPath = resultDir.buildPath("bin").absolutePath(); 115 | environment["PATH"] = binPath ~ pathSeparator ~ environment["PATH"]; 116 | 117 | version (Windows) 118 | return spawnProcess(command).wait(); 119 | else 120 | { 121 | execvp(command[0], command); 122 | errnoEnforce(false, "execvp failed"); 123 | assert(false); // unreachable 124 | } 125 | } 126 | 127 | @(`Install Digger's build result on top of an existing stable DMD installation`) 128 | int install( 129 | Switch!("Do not prompt", 'y') yes, 130 | Switch!("Only print what would be done", 'n') dryRun, 131 | Parameter!(string, "Directory to install to. Default is to find one in PATH.") installLocation = null, 132 | ) 133 | { 134 | enforce(!yes || !dryRun, "--yes and --dry-run are mutually exclusive"); 135 | .install.install(yes, dryRun, installLocation); 136 | return 0; 137 | } 138 | 139 | @(`Undo the "install" action`) 140 | int uninstall( 141 | Switch!("Only print what would be done", 'n') dryRun, 142 | Switch!("Do not verify files to be deleted; ignore errors") force, 143 | Parameter!(string, "Directory to uninstall from. Default is to search PATH.") installLocation = null, 144 | ) 145 | { 146 | .uninstall(dryRun, force, installLocation); 147 | return 0; 148 | } 149 | 150 | @(`Bisect D history according to a bisect.ini file`) 151 | int bisect( 152 | Switch!("Skip sanity-check of the GOOD/BAD commits.") noVerify, 153 | Option!(string[], "Additional bisect configuration. Equivalent to bisect.ini settings.", "NAME=VALUE", 'c', "config") configLines, 154 | Parameter!(string, "Location of the bisect.ini file containing the bisection description.") bisectConfigFile = null, 155 | ) 156 | { 157 | return doBisect(noVerify, bisectConfigFile, configLines); 158 | } 159 | 160 | @(`Cache maintenance actions (run with no arguments for details)`) 161 | int cache(string[] args) 162 | { 163 | static struct CacheActions 164 | { 165 | static: 166 | @(`Compact the cache`) 167 | int compact() 168 | { 169 | d.optimizeCache(); 170 | return 0; 171 | } 172 | 173 | @(`Delete entries cached as unbuildable`) 174 | int purgeUnbuildable() 175 | { 176 | d.purgeUnbuildable(); 177 | return 0; 178 | } 179 | 180 | @(`Migrate cached entries from one cache engine to another`) 181 | int migrate(string source, string target) 182 | { 183 | d.migrateCache(source, target); 184 | return 0; 185 | } 186 | } 187 | 188 | return funoptDispatch!CacheActions(["digger cache"] ~ args); 189 | } 190 | 191 | // hidden actions 192 | 193 | int buildAll(BuildOptions!("build", "built") options, string spec = "master") 194 | { 195 | parseBuildOptions(options); 196 | .buildAll(spec); 197 | return 0; 198 | } 199 | 200 | int delve(bool inBisect) 201 | { 202 | return doDelve(inBisect); 203 | } 204 | 205 | int parseRev(string rev) 206 | { 207 | stdout.writeln(.parseRev(rev)); 208 | return 0; 209 | } 210 | 211 | int show(string revision) 212 | { 213 | d.getMetaRepo().git.run("log", "-n1", revision); 214 | d.getMetaRepo().git.run("log", "-n1", "--pretty=format:t=%ct", revision); 215 | return 0; 216 | } 217 | 218 | int getLatest() 219 | { 220 | writeln((cast(DManager.Website)d.getComponent("website")).getLatest()); 221 | return 0; 222 | } 223 | 224 | int help() 225 | { 226 | throw new Exception("For help, run digger without any arguments."); 227 | } 228 | 229 | version (Windows) 230 | int getAllMSIs() 231 | { 232 | d.getVSInstaller().getAllMSIs(); 233 | return 0; 234 | } 235 | } 236 | 237 | int digger() 238 | { 239 | version (D_Coverage) 240 | { 241 | import core.runtime; 242 | dmd_coverSetMerge(true); 243 | } 244 | 245 | static void usageFun(string usage) 246 | { 247 | import std.algorithm, std.array, std.stdio, std.string; 248 | auto lines = usage.splitLines(); 249 | 250 | stderr.writeln("Digger v" ~ diggerVersion ~ " - a D source code building and archaeology tool"); 251 | stderr.writeln("Created by Vladimir Panteleev "); 252 | stderr.writeln("https://github.com/CyberShadow/Digger"); 253 | stderr.writeln(); 254 | stderr.writeln("Configuration file: ", opts.configFile.value.exists ? opts.configFile.value : "(not present)"); 255 | stderr.writeln("Working directory: ", config.config.local.workDir); 256 | stderr.writeln(); 257 | 258 | if (lines[0].canFind("ACTION [ACTION-ARGUMENTS]")) 259 | { 260 | lines = 261 | [lines[0].replace(" ACTION ", " [OPTION]... ACTION ")] ~ 262 | getUsageFormatString!(structFun!Opts).splitLines()[1..$] ~ 263 | lines[1..$]; 264 | 265 | stderr.writefln("%-(%s\n%)", lines); 266 | stderr.writeln(); 267 | stderr.writeln("For help on a specific action, run: digger ACTION --help"); 268 | stderr.writeln("For more information, see README.md."); 269 | stderr.writeln(); 270 | } 271 | else 272 | stderr.writefln("%-(%s\n%)", lines); 273 | } 274 | 275 | return funoptDispatch!(Digger, FunOptConfig.init, usageFun)([thisExePath] ~ (opts.action ? [opts.action.value] ~ opts.actionArguments : [])); 276 | } 277 | 278 | mixin main!digger; 279 | -------------------------------------------------------------------------------- /bisect.d: -------------------------------------------------------------------------------- 1 | module bisect; 2 | 3 | import core.thread; 4 | 5 | import std.algorithm; 6 | import std.exception; 7 | import std.file; 8 | import std.getopt : getopt; 9 | import std.path; 10 | import std.process; 11 | import std.range; 12 | import std.string; 13 | 14 | import ae.sys.file; 15 | import ae.sys.git; 16 | import ae.utils.math; 17 | import ae.utils.sini; 18 | 19 | import common; 20 | import config; 21 | import custom; 22 | import repo; 23 | 24 | enum EXIT_UNTESTABLE = 125; 25 | 26 | string bisectConfigFile; 27 | struct BisectConfig 28 | { 29 | string bad, good; 30 | bool reverse; 31 | string tester; 32 | bool bisectBuild; 33 | bool bisectBuildTest; 34 | string extraSpec; 35 | 36 | DiggerManager.Config.Build* build; 37 | DiggerManager.Config.Local* local; 38 | 39 | string[string] environment; 40 | } 41 | BisectConfig bisectConfig; 42 | 43 | /// Final build directory for bisect tests. 44 | alias currentDir = subDir!"current"; 45 | 46 | int doBisect(bool noVerify, string bisectConfigFile, string[] bisectConfigLines) 47 | { 48 | bisectConfig.build = &d.config.build; 49 | bisectConfig.local = &d.config.local; 50 | if (bisectConfigFile) 51 | { 52 | log("Loading bisect configuration from " ~ bisectConfigFile); 53 | bisectConfigFile 54 | .readText() 55 | .splitLines() 56 | .parseIniInto(bisectConfig); 57 | } 58 | else 59 | log("No bisect.ini file specified! Using options from command-line only."); 60 | bisectConfigLines.parseIniInto(bisectConfig); 61 | 62 | void ensureDefault(ref string var, string which, string def) 63 | { 64 | if (!var) 65 | { 66 | log("No %s revision specified, assuming '%s'".format(which, def)); 67 | var = def; 68 | } 69 | } 70 | ensureDefault(bisectConfig.bad , "bad" , "master"); 71 | ensureDefault(bisectConfig.good, "good", "@ 1 month ago"); 72 | 73 | if (bisectConfig.bisectBuildTest) 74 | { 75 | bisectConfig.bisectBuild = true; 76 | d.config.local.cache = "none"; 77 | } 78 | if (bisectConfig.bisectBuild) 79 | enforce(!bisectConfig.tester, "bisectBuild and specifying a test command are mutually exclusive"); 80 | enforce(bisectConfig.tester || bisectConfig.bisectBuild, "No tester specified (and bisectBuild is false)"); 81 | 82 | auto repo = &d.getMetaRepo().git(); 83 | 84 | d.needUpdate(); 85 | 86 | void test(bool good, string rev) 87 | { 88 | auto name = good ? "GOOD" : "BAD"; 89 | log("Sanity-check, testing %s revision %s...".format(name, rev)); 90 | auto result = doBisectStep(rev); 91 | enforce(result != EXIT_UNTESTABLE, 92 | "%s revision %s is not testable" 93 | .format(name, rev)); 94 | enforce(!result == good, 95 | "%s revision %s is not correct (exit status is %d)" 96 | .format(name, rev, result)); 97 | } 98 | 99 | if (!noVerify) 100 | { 101 | auto good = getRev!true(); 102 | auto bad = getRev!false(); 103 | 104 | enforce(good != bad, "Good and bad revisions are both " ~ bad); 105 | 106 | auto commonAncestor = repo.query("merge-base", good, bad); 107 | if (bisectConfig.reverse) 108 | { 109 | enforce(good != commonAncestor, "Bad commit is newer than good commit (and reverse search is enabled)"); 110 | test(false, bad); 111 | test(true, good); 112 | } 113 | else 114 | { 115 | enforce(bad != commonAncestor, "Good commit is newer than bad commit"); 116 | test(true, good); 117 | test(false, bad); 118 | } 119 | } 120 | 121 | auto p0 = getRev!true(); // good 122 | auto p1 = getRev!false(); // bad 123 | if (bisectConfig.reverse) 124 | swap(p0, p1); 125 | 126 | bool[string] cacheState, untestable; 127 | if (!bisectConfig.extraSpec) 128 | cacheState = d.getCacheState([p0, p1]); 129 | 130 | bisectLoop: 131 | while (true) 132 | { 133 | log("Finding shortest path between %s and %s...".format(p0, p1)); 134 | auto fullPath = repo.pathBetween(p0, p1); // closed interval 135 | enforce(fullPath.length >= 2 && fullPath[0].commit == p0 && fullPath[$-1].commit == p1, 136 | "Bad path calculation result"); 137 | auto path = fullPath[1..$-1].map!(step => step.commit).array; // open interval 138 | log("%d commits (about %d tests) remaining.".format(path.length, ilog2(path.length+1))); 139 | 140 | if (!path.length) 141 | { 142 | assert(fullPath.length == 2); 143 | auto p = fullPath[1].downwards ? p0 : p1; 144 | log("%s is the first %s commit".format(p, bisectConfig.reverse ? "good" : "bad")); 145 | repo.run("--no-pager", "show", p); 146 | log("Bisection completed successfully."); 147 | return 0; 148 | } 149 | 150 | log("(%d total, %d cached, %d untestable)".format( 151 | path.length, 152 | path.filter!(commit => cacheState.get(commit, false)).walkLength, 153 | path.filter!(commit => commit in untestable).walkLength, 154 | )); 155 | 156 | // First try all cached commits in the range (middle-most first). 157 | // Afterwards, do a binary-log search across the commit range for a testable commit. 158 | auto order = chain( 159 | path.radial .filter!(commit => cacheState.get(commit, false)), 160 | path.binaryOrder.filter!(commit => !cacheState.get(commit, false)) 161 | ).filter!(commit => commit !in untestable).array; 162 | 163 | foreach (i, p; order) 164 | { 165 | auto result = doBisectStep(p); 166 | if (result == EXIT_UNTESTABLE) 167 | { 168 | log("Commit %s (%d/%d) is untestable.".format(p, i+1, order.length)); 169 | untestable[p] = true; 170 | continue; 171 | } 172 | 173 | if (bisectConfig.reverse) 174 | result = result ? 0 : 1; 175 | 176 | if (result == 0) // good 177 | p0 = p; 178 | else 179 | p1 = p; 180 | 181 | continue bisectLoop; 182 | } 183 | 184 | log("There are only untestable commits left to bisect."); 185 | log("The first %s commit could be any of:".format(bisectConfig.reverse ? "good" : "bad")); 186 | foreach (p; path ~ [p1]) 187 | repo.run("log", "-1", "--pretty=format:%h %ci: %s", p); 188 | log("We cannot bisect more!"); 189 | return 2; 190 | } 191 | 192 | assert(false); 193 | } 194 | 195 | struct BisectStep 196 | { 197 | string commit; 198 | bool downwards; // on the way to the common ancestor 199 | } 200 | 201 | BisectStep[] pathBetween(in Repository* repo, string p0, string p1) 202 | { 203 | auto commonAncestor = repo.query("merge-base", p0, p1); 204 | return chain( 205 | repo.commitsBetween(commonAncestor, p0).retro.map!(commit => BisectStep(commit, true )), 206 | commonAncestor.only .map!(commit => BisectStep(commit, true )), 207 | repo.commitsBetween(commonAncestor, p1) .map!(commit => BisectStep(commit, false)), 208 | ).array; 209 | } 210 | 211 | string[] commitsBetween(in Repository* repo, string p0, string p1) 212 | { 213 | return repo.query("log", "--reverse", "--pretty=format:%H", p0 ~ ".." ~ p1).splitLines(); 214 | } 215 | 216 | /// Reorders [1, 2, ..., 98, 99] 217 | /// into [50, 25, 75, 13, 38, 63, 88, ...] 218 | T[] binaryOrder(T)(T[] items) 219 | { 220 | auto n = items.length; 221 | assert(n); 222 | auto seen = new bool[n]; 223 | auto result = new T[n]; 224 | size_t c = 0; 225 | 226 | foreach (p; 0..30) 227 | foreach (i; 0..1<= n || seen[x]) 231 | continue; 232 | seen[x] = true; 233 | result[c++] = items[x]; 234 | if (c == n) 235 | return result; 236 | } 237 | assert(false); 238 | } 239 | 240 | unittest 241 | { 242 | assert(iota(1, 7+1).array.binaryOrder.equal([4, 2, 6, 1, 3, 5, 7])); 243 | assert(iota(1, 100).array.binaryOrder.startsWith([50, 25, 75, 13, 38, 63, 88])); 244 | } 245 | 246 | int doBisectStep(string rev) 247 | { 248 | log("Testing revision: " ~ rev); 249 | 250 | try 251 | { 252 | if (currentDir.exists) 253 | { 254 | version (Windows) 255 | { 256 | try 257 | currentDir.rmdirRecurse(); 258 | catch (Exception e) 259 | { 260 | log("Failed to clean up %s: %s".format(currentDir, e.msg)); 261 | Thread.sleep(500.msecs); 262 | log("Retrying..."); 263 | currentDir.rmdirRecurse(); 264 | } 265 | } 266 | else 267 | currentDir.rmdirRecurse(); 268 | } 269 | 270 | auto state = parseSpec(rev ~ bisectConfig.extraSpec); 271 | 272 | scope (exit) 273 | if (d.buildDir.exists) 274 | rename(d.buildDir, currentDir); 275 | 276 | d.build(state); 277 | } 278 | catch (Exception e) 279 | { 280 | log("Build failed: " ~ e.toString()); 281 | if (bisectConfig.bisectBuild && !bisectConfig.bisectBuildTest) 282 | return 1; 283 | return EXIT_UNTESTABLE; 284 | } 285 | 286 | if (bisectConfig.bisectBuild) 287 | { 288 | log("Build successful."); 289 | 290 | if (bisectConfig.bisectBuildTest) 291 | { 292 | try 293 | d.test(); 294 | catch (Exception e) 295 | { 296 | log("Tests failed: " ~ e.toString()); 297 | return 1; 298 | } 299 | log("Tests successful."); 300 | } 301 | 302 | return 0; 303 | } 304 | 305 | string[string] env = d.getBaseEnvironment(); 306 | d.applyEnv(env, bisectConfig.environment); 307 | 308 | auto oldPath = environment["PATH"]; 309 | scope(exit) environment["PATH"] = oldPath; 310 | 311 | // Add the final DMD to the environment PATH 312 | env["PATH"] = buildPath(currentDir, "bin").absolutePath() ~ pathSeparator ~ env["PATH"]; 313 | environment["PATH"] = env["PATH"]; 314 | 315 | // Use host HOME for the test command 316 | env["HOME"] = environment.get("HOME"); 317 | 318 | // For bisecting bootstrapping issues - allows passing the revision to another Digger instance 319 | env["DIGGER_REVISION"] = rev; 320 | 321 | d.logProgress("Running test command..."); 322 | auto result = spawnShell(bisectConfig.tester, env, Config.newEnv).wait(); 323 | d.logProgress("Test command exited with status %s (%s).".format(result, result==0 ? "GOOD" : result==EXIT_UNTESTABLE ? "UNTESTABLE" : "BAD")); 324 | return result; 325 | } 326 | 327 | /// Returns SHA-1 of the initial search points. 328 | string getRev(bool good)() 329 | { 330 | static string result; 331 | if (!result) 332 | { 333 | auto rev = good ? bisectConfig.good : bisectConfig.bad; 334 | result = parseRev(rev); 335 | log("Resolved %s revision `%s` to %s.".format(good ? "GOOD" : "BAD", rev, result)); 336 | } 337 | return result; 338 | } 339 | 340 | struct CommitRange 341 | { 342 | uint startTime; /// first bad commit 343 | uint endTime; /// first good commit 344 | } 345 | /// Known unbuildable time ranges 346 | const CommitRange[] badCommits = 347 | [ 348 | { 1342243766, 1342259226 }, // Messed up DMD make files 349 | { 1317625155, 1319346272 }, // Missing std.stdio import in std.regex 350 | ]; 351 | 352 | /// Find the earliest revision that Digger can build. 353 | /// Used during development to extend Digger's range. 354 | int doDelve(bool inBisect) 355 | { 356 | if (inBisect) 357 | { 358 | log("Invoked by git-bisect - performing bisect step."); 359 | 360 | import std.conv; 361 | auto rev = d.getMetaRepo().getRef("BISECT_HEAD"); 362 | auto t = d.getMetaRepo().git.query("log", "-n1", "--pretty=format:%ct", rev).to!int(); 363 | foreach (r; badCommits) 364 | if (r.startTime <= t && t < r.endTime) 365 | { 366 | log("This revision is known to be unbuildable, skipping."); 367 | return EXIT_UNTESTABLE; 368 | } 369 | 370 | d.cacheFailures = false; 371 | //d.config.build = bisectConfig.build; // TODO 372 | auto state = parseSpec(rev ~ bisectConfig.extraSpec); 373 | try 374 | { 375 | d.build(state); 376 | return 1; 377 | } 378 | catch (Exception e) 379 | { 380 | log("Build failed: " ~ e.toString()); 381 | return 0; 382 | } 383 | } 384 | else 385 | { 386 | auto root = d.getMetaRepo().git.query("log", "--pretty=format:%H", "--reverse", "master").splitLines()[0]; 387 | d.getMetaRepo().git.run(["bisect", "start", "--no-checkout", "master", root]); 388 | d.getMetaRepo().git.run("bisect", "run", 389 | thisExePath, 390 | "--dir", getcwd(), 391 | "--config-file", opts.configFile, 392 | "delve", "--in-bisect", 393 | ); 394 | return 0; 395 | } 396 | } 397 | -------------------------------------------------------------------------------- /install.d: -------------------------------------------------------------------------------- 1 | module install; 2 | 3 | import std.algorithm; 4 | import std.array; 5 | import std.exception; 6 | import std.file; 7 | import std.path; 8 | import std.string; 9 | 10 | import ae.sys.file; 11 | import ae.utils.array; 12 | import ae.utils.json; 13 | 14 | import common; 15 | import config : config; 16 | import custom; 17 | 18 | static import std.process; 19 | 20 | alias copy = std.file.copy; // https://issues.dlang.org/show_bug.cgi?id=14817 21 | 22 | version (Windows) 23 | { 24 | enum string binExt = ".exe"; 25 | enum string dmdConfigName = "sc.ini"; 26 | } 27 | else 28 | { 29 | enum string binExt = ""; 30 | enum string dmdConfigName = "dmd.conf"; 31 | } 32 | 33 | version (Posix) 34 | { 35 | import core.sys.posix.unistd; 36 | import std.conv : octal; 37 | } 38 | 39 | version (Windows) 40 | enum string platformDir = "windows"; 41 | else 42 | version (linux) 43 | enum string platformDir = "linux"; 44 | else 45 | version (OSX) 46 | enum string platformDir = "osx"; 47 | else 48 | version (FreeBSD) 49 | enum string platformDir = "freebsd"; 50 | else 51 | enum string platformDir = null; 52 | 53 | string[] findDMD() 54 | { 55 | string[] result; 56 | foreach (pathEntry; std.process.environment.get("PATH", null).split(pathSeparator)) 57 | { 58 | auto dmd = pathEntry.buildPath("dmd" ~ binExt); 59 | if (dmd.exists) 60 | result ~= dmd; 61 | } 62 | return result; 63 | } 64 | 65 | string selectInstallPath(string location) 66 | { 67 | string[] candidates; 68 | if (location) 69 | { 70 | auto dmd = location.absolutePath(); 71 | @property bool ok() { return dmd.exists && dmd.isFile; } 72 | 73 | if (!ok) 74 | { 75 | string newDir; 76 | 77 | bool dirOK(string dir) 78 | { 79 | newDir = dmd.buildPath(dir); 80 | return newDir.exists && newDir.isDir; 81 | } 82 | 83 | bool tryDir(string dir) 84 | { 85 | if (dirOK(dir)) 86 | { 87 | dmd = newDir; 88 | return true; 89 | } 90 | return false; 91 | } 92 | 93 | tryDir("dmd2"); 94 | 95 | static if (platformDir) 96 | tryDir(platformDir); 97 | 98 | enforce(!dirOK("bin32") || !dirOK("bin64"), 99 | "Ambiguous model in path - please specify full path to DMD binary"); 100 | tryDir("bin") || tryDir("bin32") || tryDir("bin64"); 101 | 102 | dmd = dmd.buildPath("dmd" ~ binExt); 103 | enforce(ok, "DMD installation not detected at " ~ location); 104 | } 105 | 106 | candidates = [dmd]; 107 | } 108 | else 109 | { 110 | candidates = findDMD(); 111 | enforce(candidates.length, "DMD not found in PATH - " ~ 112 | "add DMD to PATH or specify install location explicitly"); 113 | } 114 | 115 | foreach (candidate; candidates) 116 | { 117 | if (candidate.buildNormalizedPath.startsWith(config.local.workDir.buildNormalizedPath)) 118 | { 119 | log("Skipping DMD installation under Digger workDir (" ~ config.local.workDir ~ "): " ~ candidate); 120 | continue; 121 | } 122 | 123 | log("Found DMD executable: " ~ candidate); 124 | return candidate; 125 | } 126 | 127 | throw new Exception("No suitable DMD installation found."); 128 | } 129 | 130 | string findConfig(string dmdPath) 131 | { 132 | string configPath; 133 | 134 | bool pathOK(string path) 135 | { 136 | configPath = path.buildPath(dmdConfigName); 137 | return configPath.exists; 138 | } 139 | 140 | import std.process : execute; 141 | auto result = execute([dmdPath, "--help"]); 142 | if (result.status == 0) 143 | foreach (line; result.output.splitLines) 144 | if (line.skipOver("Config file: ")) 145 | if (line.exists) 146 | return line; 147 | 148 | if (pathOK(dmdPath.dirName)) 149 | return configPath; 150 | 151 | auto home = std.process.environment.get("HOME", null); 152 | if (home && pathOK(home)) 153 | return configPath; 154 | 155 | version (Posix) 156 | if (pathOK("/etc/")) 157 | return configPath; 158 | 159 | throw new Exception(format("Can't find DMD configuration file %s " ~ 160 | "corresponding to DMD located at %s", dmdConfigName, dmdPath)); 161 | } 162 | 163 | struct ComponentPaths 164 | { 165 | string binPath; 166 | string libPath; 167 | string phobosPath; 168 | string druntimePath; 169 | } 170 | 171 | string commonModel(ref BuildInfo buildInfo) 172 | { 173 | enforce(buildInfo.config.components.common.models.length == 1, "Multi-model install is not yet supported"); 174 | return buildInfo.config.components.common.models[0]; 175 | } 176 | 177 | ComponentPaths parseConfig(string dmdPath, BuildInfo buildInfo) 178 | { 179 | auto configPath = findConfig(dmdPath); 180 | log("Found DMD configuration: " ~ configPath); 181 | 182 | string[string] vars = std.process.environment.toAA(); 183 | bool parsing = false; 184 | foreach (line; configPath.readText().splitLines()) 185 | { 186 | if (line.startsWith("[") && line.endsWith("]")) 187 | { 188 | auto sectionName = line[1..$-1]; 189 | parsing = sectionName == "Environment" 190 | || sectionName == "Environment" ~ buildInfo.commonModel; 191 | } 192 | else 193 | if (parsing && line.canFind("=")) 194 | { 195 | string name, value; 196 | list(name, null, value) = line.findSplit("="); 197 | auto parts = value.split("%"); 198 | for (size_t n = 1; n < parts.length; n+=2) 199 | if (!parts[n].length) 200 | parts[n] = "%"; 201 | else 202 | if (parts[n] == "@P") 203 | parts[n] = configPath.dirName(); 204 | else 205 | parts[n] = vars.get(parts[n], parts[n]); 206 | value = parts.join(); 207 | vars[name] = value; 208 | } 209 | } 210 | 211 | string[] parseParameters(string s, char escape = '\\', char separator = ' ') 212 | { 213 | string[] result; 214 | while (s.length) 215 | if (s[0] == separator) 216 | s = s[1..$]; 217 | else 218 | { 219 | string p; 220 | if (s[0] == '"') 221 | { 222 | s = s[1..$]; 223 | bool escaping, end; 224 | while (s.length && !end) 225 | { 226 | auto c = s[0]; 227 | s = s[1..$]; 228 | if (!escaping && c == '"') 229 | end = true; 230 | else 231 | if (!escaping && c == escape) 232 | escaping = true; 233 | else 234 | { 235 | if (escaping && c != escape) 236 | p ~= escape; 237 | p ~= c; 238 | escaping = false; 239 | } 240 | } 241 | } 242 | else 243 | list(p, null, s) = s.findSplit([separator]); 244 | result ~= p; 245 | } 246 | return result; 247 | } 248 | 249 | string[] dflags = parseParameters(vars.get("DFLAGS", null)); 250 | string[] importPaths = dflags 251 | .filter!(s => s.startsWith("-I")) 252 | .map!(s => s[2..$].split(";")) 253 | .join(); 254 | 255 | version (Windows) 256 | string[] libPaths = parseParameters(vars.get("LIB", null), 0, ';'); 257 | else 258 | string[] libPaths = dflags 259 | .filter!(s => s.startsWith("-L-L")) 260 | .map!(s => s[4..$]) 261 | .array(); 262 | 263 | string findPath(string[] paths, string name, string[] testFiles) 264 | { 265 | auto results = paths.find!(path => testFiles.any!(testFile => path.buildPath(testFile).exists)); 266 | enforce(!results.empty, "Can't find %s (%-(%s or %)). Looked in: %s".format(name, testFiles, paths)); 267 | auto result = results.front.buildNormalizedPath(); 268 | auto testFile = testFiles.find!(testFile => result.buildPath(testFile).exists).front; 269 | log("Found %s (%s): %s".format(name, testFile, result)); 270 | return result; 271 | } 272 | 273 | ComponentPaths result; 274 | result.binPath = dmdPath.dirName(); 275 | result.libPath = findPath(libPaths, "Phobos static library", [getLibFileName(buildInfo)]); 276 | result.phobosPath = findPath(importPaths, "Phobos source code", ["std/stdio.d"]); 277 | result.druntimePath = findPath(importPaths, "Druntime import files", ["object.d", "object.di"]); 278 | return result; 279 | } 280 | 281 | string getLibFileName(BuildInfo buildInfo) 282 | { 283 | version (Windows) 284 | { 285 | auto model = buildInfo.commonModel; 286 | return "phobos%s.lib".format(model == "32" ? "" : model); 287 | } 288 | else 289 | return "libphobos2.a"; 290 | } 291 | 292 | struct InstalledObject 293 | { 294 | /// File name in backup directory 295 | string name; 296 | 297 | /// Original location. 298 | /// Path is relative to uninstall.json's directory. 299 | string path; 300 | 301 | /// MD5 sum of the NEW object's contents 302 | /// (not the one in the install directory). 303 | /// For directories, this is the MD5 sum 304 | /// of all files sorted by name (see mdDir). 305 | string hash; 306 | } 307 | 308 | struct UninstallData 309 | { 310 | InstalledObject*[] objects; 311 | } 312 | 313 | void install(bool yes, bool dryRun, string location = null) 314 | { 315 | assert(!yes || !dryRun, "Mutually exclusive options"); 316 | auto dmdPath = selectInstallPath(location); 317 | 318 | auto buildInfoPath = resultDir.buildPath(buildInfoFileName); 319 | enforce(buildInfoPath.exists, 320 | buildInfoPath ~ " not found - please purge cache and rebuild"); 321 | auto buildInfo = buildInfoPath.readText().jsonParse!BuildInfo(); 322 | 323 | auto componentPaths = parseConfig(dmdPath, buildInfo); 324 | 325 | auto verb = dryRun ? "Would" : "Will"; 326 | log("%s install:".format(verb)); 327 | log(" - Binaries to: " ~ componentPaths.binPath); 328 | log(" - Libraries to: " ~ componentPaths.libPath); 329 | log(" - Phobos source code to: " ~ componentPaths.phobosPath); 330 | log(" - Druntime includes to: " ~ componentPaths.druntimePath); 331 | 332 | auto uninstallPath = buildPath(componentPaths.binPath, ".digger-install"); 333 | auto uninstallFileName = buildPath(uninstallPath, "uninstall.json"); 334 | bool updating = uninstallFileName.exists; 335 | if (updating) 336 | { 337 | log("Found previous installation data in " ~ uninstallPath); 338 | log("%s update existing installation.".format(verb)); 339 | } 340 | else 341 | { 342 | log("This %s be a new Digger installation.".format(verb.toLower)); 343 | log("Backups and uninstall data %s be saved in %s".format(verb.toLower, uninstallPath)); 344 | } 345 | 346 | auto libFileName = getLibFileName(buildInfo); 347 | auto libName = libFileName.stripExtension ~ "-" ~ buildInfo.commonModel ~ libFileName.extension; 348 | 349 | static struct Item 350 | { 351 | string name, srcPath, dstPath; 352 | } 353 | 354 | Item[] items = 355 | [ 356 | Item("dmd" ~ binExt, buildPath(resultDir, "bin", "dmd" ~ binExt), dmdPath), 357 | Item("rdmd" ~ binExt, buildPath(resultDir, "bin", "rdmd" ~ binExt), buildPath(componentPaths.binPath, "rdmd" ~ binExt)), 358 | Item(libName , buildPath(resultDir, "lib", libFileName) , buildPath(componentPaths.libPath, libFileName)), 359 | Item("object.di" , buildPath(resultDir, "import", "object.{d,di}").globFind, 360 | buildPath(componentPaths.druntimePath, "object.{d,di}").globFind), 361 | Item("core" , buildPath(resultDir, "import", "core") , buildPath(componentPaths.druntimePath, "core")), 362 | Item("std" , buildPath(resultDir, "import", "std") , buildPath(componentPaths.phobosPath, "std")), 363 | Item("etc" , buildPath(resultDir, "import", "etc") , buildPath(componentPaths.phobosPath, "etc")), 364 | ]; 365 | 366 | InstalledObject*[string] existingComponents; 367 | bool[string] updateNeeded; 368 | 369 | UninstallData uninstallData; 370 | 371 | if (updating) 372 | { 373 | uninstallData = uninstallFileName.readText.jsonParse!UninstallData; 374 | foreach (obj; uninstallData.objects) 375 | existingComponents[obj.name] = obj; 376 | } 377 | 378 | log("Preparing object list..."); 379 | 380 | foreach (item; items) 381 | { 382 | log(" - " ~ item.name); 383 | 384 | auto obj = new InstalledObject(item.name, item.dstPath.relativePath(uninstallPath), mdObject(item.srcPath)); 385 | auto pexistingComponent = item.name in existingComponents; 386 | if (pexistingComponent) 387 | { 388 | auto existingComponent = *pexistingComponent; 389 | 390 | enforce(existingComponent.path == obj.path, 391 | "Updated component has a different path (%s vs %s), aborting." 392 | .format(existingComponents[item.name].path, obj.path)); 393 | 394 | verifyObject(existingComponent, uninstallPath, "update"); 395 | 396 | updateNeeded[item.name] = existingComponent.hash != obj.hash; 397 | existingComponent.hash = obj.hash; 398 | } 399 | else 400 | uninstallData.objects ~= obj; 401 | } 402 | 403 | log("Testing write access and filesystem boundaries:"); 404 | 405 | string[] dirs = items.map!(item => item.dstPath.dirName).array.sort().uniq().array; 406 | foreach (dir; dirs) 407 | { 408 | log(" - %s".format(dir)); 409 | auto testPathA = dir.buildPath(".digger-test"); 410 | auto testPathB = componentPaths.binPath.buildPath(".digger-test2"); 411 | 412 | std.file.write(testPathA, "test"); 413 | { 414 | scope(failure) remove(testPathA); 415 | rename(testPathA, testPathB); 416 | } 417 | remove(testPathB); 418 | } 419 | 420 | version (Posix) 421 | { 422 | int owner = dmdPath.getOwner(); 423 | int group = dmdPath.getGroup(); 424 | int mode = items.front.dstPath.getAttributes() & octal!666; 425 | log("UID=%d GID=%d Mode=%03o".format(owner, group, mode)); 426 | } 427 | 428 | log("Things to do:"); 429 | 430 | foreach (item; items) 431 | { 432 | enforce(item.srcPath.exists, "Can't find source for component %s: %s".format(item.name, item.srcPath)); 433 | enforce(item.dstPath.exists, "Can't find target for component %s: %s".format(item.name, item.dstPath)); 434 | 435 | string action; 436 | if (updating && item.name in existingComponents) 437 | action = updateNeeded[item.name] ? "Update" : "Skip unchanged"; 438 | else 439 | action = "Install"; 440 | log(" - %s component %s from %s to %s".format(action, item.name, item.srcPath, item.dstPath)); 441 | } 442 | 443 | log("You %s be able to undo this action by running `digger uninstall`.".format(verb.toLower)); 444 | 445 | if (dryRun) 446 | { 447 | log("Dry run, exiting."); 448 | return; 449 | } 450 | 451 | if (yes) 452 | log("Proceeding with installation."); 453 | else 454 | { 455 | import std.stdio : stdin, stderr; 456 | 457 | string result; 458 | do 459 | { 460 | stderr.write("Proceed with installation? [Y/n] "); stderr.flush(); 461 | result = stdin.readln().chomp().toLower(); 462 | } while (result != "y" && result != "n" && result != ""); 463 | if (result == "n") 464 | return; 465 | } 466 | 467 | enforce(updating || !uninstallPath.exists, "Uninstallation directory exists without uninstall.json: " ~ uninstallPath); 468 | 469 | log("Saving uninstall information..."); 470 | 471 | if (!updating) 472 | mkdir(uninstallPath); 473 | std.file.write(uninstallFileName, toJson(uninstallData)); 474 | 475 | log("Backing up original files..."); 476 | 477 | foreach (item; items) 478 | if (item.name !in existingComponents) 479 | { 480 | log(" - " ~ item.name); 481 | auto backupPath = buildPath(uninstallPath, item.name); 482 | rename(item.dstPath, backupPath); 483 | } 484 | 485 | if (updating) 486 | { 487 | log("Cleaning up existing Digger-installed files..."); 488 | 489 | foreach (item; items) 490 | if (item.name in existingComponents && updateNeeded[item.name]) 491 | { 492 | log(" - " ~ item.name); 493 | rmObject(item.dstPath); 494 | } 495 | } 496 | 497 | log("Installing new files..."); 498 | 499 | foreach (item; items) 500 | if (item.name !in existingComponents || updateNeeded[item.name]) 501 | { 502 | log(" - " ~ item.name); 503 | atomic!cpObject(item.srcPath, item.dstPath); 504 | } 505 | 506 | version (Posix) 507 | { 508 | log("Applying attributes..."); 509 | 510 | bool isRoot = geteuid()==0; 511 | 512 | foreach (item; items) 513 | if (item.name !in existingComponents || updateNeeded[item.name]) 514 | { 515 | log(" - " ~ item.name); 516 | item.dstPath.recursive!setMode(mode); 517 | 518 | if (isRoot) 519 | item.dstPath.recursive!setOwner(owner, group); 520 | else 521 | if (item.dstPath.getOwner() != owner || item.dstPath.getGroup() != group) 522 | log("Warning: UID/GID mismatch for " ~ item.dstPath); 523 | } 524 | } 525 | 526 | log("Install OK."); 527 | log("You can undo this action by running `digger uninstall`."); 528 | } 529 | 530 | void uninstall(bool dryRun, bool force, string location = null) 531 | { 532 | string uninstallPath; 533 | if (location.canFind(".digger-install")) 534 | uninstallPath = location; 535 | else 536 | { 537 | auto dmdPath = selectInstallPath(location); 538 | auto binPath = dmdPath.dirName(); 539 | uninstallPath = buildPath(binPath, ".digger-install"); 540 | } 541 | auto uninstallFileName = buildPath(uninstallPath, "uninstall.json"); 542 | enforce(uninstallFileName.exists, "Can't find uninstallation data: " ~ uninstallFileName); 543 | auto uninstallData = uninstallFileName.readText.jsonParse!UninstallData; 544 | 545 | if (!force) 546 | { 547 | log("Verifying files to be uninstalled..."); 548 | 549 | foreach (obj; uninstallData.objects) 550 | verifyObject(obj, uninstallPath, "uninstall"); 551 | 552 | log("Verify OK."); 553 | } 554 | 555 | log(dryRun ? "Actions to run:" : "Uninstalling..."); 556 | 557 | void runAction(void delegate() action) 558 | { 559 | if (!force) 560 | action(); 561 | else 562 | try 563 | action(); 564 | catch (Exception e) 565 | log("Ignoring error: " ~ e.msg); 566 | } 567 | 568 | void uninstallObject(InstalledObject* obj) 569 | { 570 | auto src = buildNormalizedPath(uninstallPath, obj.name); 571 | auto dst = buildNormalizedPath(uninstallPath, obj.path); 572 | 573 | if (!src.exists) // --force 574 | { 575 | log(" - %s component %s with no backup".format(dryRun ? "Would skip" : "Skipping", obj.name)); 576 | return; 577 | } 578 | 579 | if (dryRun) 580 | { 581 | log(" - Would remove " ~ dst); 582 | log(" Would move " ~ src ~ " to " ~ dst); 583 | } 584 | else 585 | { 586 | log(" - Removing " ~ dst); 587 | runAction({ rmObject(dst); }); 588 | log(" Moving " ~ src ~ " to " ~ dst); 589 | runAction({ rename(src, dst); }); 590 | } 591 | } 592 | 593 | foreach (obj; uninstallData.objects) 594 | runAction({ uninstallObject(obj); }); 595 | 596 | if (dryRun) 597 | return; 598 | 599 | remove(uninstallFileName); 600 | 601 | if (!force) 602 | rmdir(uninstallPath); // should be empty now 603 | else 604 | rmdirRecurse(uninstallPath); 605 | 606 | log("Uninstall OK."); 607 | } 608 | 609 | string globFind(string path) 610 | { 611 | auto results = dirEntries(path.dirName, path.baseName, SpanMode.shallow); 612 | enforce(!results.empty, "Can't find: " ~ path); 613 | auto result = results.front; 614 | results.popFront(); 615 | enforce(results.empty, "Multiple matches: " ~ path); 616 | return result; 617 | } 618 | 619 | void verifyObject(InstalledObject* obj, string uninstallPath, string verb) 620 | { 621 | auto path = buildNormalizedPath(uninstallPath, obj.path); 622 | enforce(path.exists, "Can't find item to %s: %s".format(verb, path)); 623 | auto hash = mdObject(path); 624 | enforce(hash == obj.hash, 625 | "Object changed since it was installed: %s\nPlease %s manually.".format(path, verb)); 626 | } 627 | 628 | version(Posix) bool attrIsExec(int attr) { return (attr & octal!111) != 0; } 629 | 630 | /// Set access modes while preserving executable bit. 631 | version(Posix) 632 | void setMode(string fn, int mode) 633 | { 634 | auto attr = fn.getAttributes(); 635 | mode |= attr & ~octal!777; 636 | if (attr.attrIsExec || attr.attrIsDir) 637 | mode = mode | ((mode & octal!444) >> 2); // executable iff readable 638 | fn.setAttributes(mode); 639 | } 640 | 641 | /// Apply a function recursively to all files and directories under given path. 642 | template recursive(alias fun) 643 | { 644 | void recursive(Args...)(string fn, auto ref Args args) 645 | { 646 | fun(fn, args); 647 | if (fn.isDir) 648 | foreach (de; fn.dirEntries(SpanMode.shallow)) 649 | recursive(de.name, args); 650 | } 651 | } 652 | 653 | void rmObject(string path) { path.isDir ? path.rmdirRecurse() : path.remove(); } 654 | 655 | void cpObject(string src, string dst) 656 | { 657 | if (src.isDir) 658 | { 659 | mkdir(dst); 660 | dst.setAttributes(src.getAttributes()); 661 | foreach (de; src.dirEntries(SpanMode.shallow)) 662 | cpObject(de.name, buildPath(dst, de.baseName)); 663 | } 664 | else 665 | { 666 | src.copy(dst); 667 | dst.setAttributes(src.getAttributes()); 668 | } 669 | } 670 | 671 | string mdDir(string dir) 672 | { 673 | import std.stdio : File; 674 | import std.digest.md; 675 | 676 | auto dataChunks = dir 677 | .dirEntries(SpanMode.breadth) 678 | .filter!(de => de.isFile) 679 | .map!(de => de.name.replace(`\`, `/`)) 680 | .array() 681 | .sort() 682 | .map!(name => File(name, "rb").byChunk(4096)) 683 | .joiner(); 684 | 685 | MD5 digest; 686 | digest.start(); 687 | foreach (chunk; dataChunks) 688 | digest.put(chunk); 689 | auto result = digest.finish(); 690 | // https://issues.dlang.org/show_bug.cgi?id=9279 691 | auto str = result.toHexString(); 692 | return str[].idup; 693 | } 694 | 695 | string mdObject(string path) 696 | { 697 | static if (__VERSION__ < 2080) 698 | import std.digest.digest : toHexString; 699 | else 700 | import std.digest : toHexString; 701 | 702 | if (path.isDir) 703 | return path.mdDir(); 704 | else 705 | { 706 | auto result = path.mdFile(); 707 | // https://issues.dlang.org/show_bug.cgi?id=9279 708 | auto str = result.toHexString(); 709 | return str[].idup; 710 | } 711 | } 712 | --------------------------------------------------------------------------------