├── .gitattributes
├── .github
├── ISSUE_TEMPLATE
│ └── ---bug-report.md
├── dependabot.yml
└── workflows
│ └── ci.yml
├── .gitignore
├── LICENSE
├── README.md
├── build.zig
├── build.zig.zon
└── src
├── args.zig
├── env.zig
├── main.zig
├── search.zig
├── thread_pool.zig
└── utils.zig
/.gitattributes:
--------------------------------------------------------------------------------
1 | *.zig text eol=lf
2 | *.zig.zon text eol=lf
3 |
--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/---bug-report.md:
--------------------------------------------------------------------------------
1 | ---
2 | name: "\U0001F41B Bug report"
3 | about: Report a bug
4 | title: ""
5 | labels: bug
6 | assignees: ""
7 | ---
8 |
9 | ## Description
10 |
11 |
12 |
13 | ## Logs
14 |
15 | The offending command causing the error with the `$env:SCOOP_SEARCH_VERBOSE=1` env var set:
16 |
17 |
18 | Output
19 |
20 | ```txt
21 | ```
22 |
23 |
--------------------------------------------------------------------------------
/.github/dependabot.yml:
--------------------------------------------------------------------------------
1 | version: 2
2 | updates:
3 | - package-ecosystem: "github-actions"
4 | directory: "/"
5 | schedule:
6 | interval: "monthly"
7 |
--------------------------------------------------------------------------------
/.github/workflows/ci.yml:
--------------------------------------------------------------------------------
1 | name: ci
2 |
3 | on:
4 | push:
5 | tags:
6 | - "v*.*.*"
7 | branches: [master]
8 | pull_request:
9 | branches: [master]
10 |
11 | jobs:
12 | build:
13 | name: Build
14 | runs-on: windows-latest
15 | steps:
16 | - uses: actions/checkout@v4
17 |
18 | - name: Setup Zig
19 | uses: goto-bus-stop/setup-zig@v2
20 | with:
21 | version: 0.14.1
22 |
23 | - name: Check formatting
24 | run: zig fmt --check .
25 |
26 | - name: Build for Windows
27 | run: zig build -Doptimize=ReleaseFast -Dcpu=baseline
28 |
29 | - name: Release
30 | uses: softprops/action-gh-release@v2
31 | if: startsWith(github.ref, 'refs/tags/') && github.repository == 'shilangyu/scoop-search'
32 | with:
33 | files: |
34 | zig-out/bin/scoop-search.exe
35 | env:
36 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
37 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | tests/*.out
2 | .zig-cache/
3 | zig-out/
4 | testing/
5 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2020 Marcin Wojnarowski
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # scoop-search
2 |
3 | [](https://github.com/shilangyu/scoop-search/actions)
4 |
5 | Fast `scoop search` drop-in replacement 🚀
6 |
7 | ## Installation
8 |
9 | ```sh
10 | scoop install scoop-search
11 | ```
12 |
13 | ## PowerShell hook
14 |
15 | Instead of using `scoop-search.exe ` you can create a hook that will run `scoop-search.exe` whenever you use native `scoop search`
16 |
17 | Add this to your Powershell profile (usually located at `$PROFILE`)
18 |
19 | ```ps1
20 | Invoke-Expression (&scoop-search --hook)
21 | ```
22 |
23 | ## CMD.exe wrapper
24 |
25 | If you use `cmd.exe` you can use a wrapper script to do the same. Name this `scoop.cmd` and add it to
26 | a directory in your `%PATH%`
27 |
28 | ```
29 | @echo off
30 |
31 | if "%1" == "search" (
32 | call :search_subroutine %*
33 | ) else (
34 | powershell scoop.ps1 %*
35 | )
36 | goto :eof
37 |
38 | :search_subroutine
39 | set "args=%*"
40 | set "newargs=%args:* =%"
41 | scoop-search.exe %newargs%
42 | goto :eof
43 | ```
44 |
45 | ## Features
46 |
47 | Behaves just like `scoop search` and returns identical output. If any differences are found please open an issue.
48 |
49 | **Non-goal**: any additional features unavailable in scoop search
50 |
51 | ## Building
52 |
53 | This project uses Zig. Building and running works on all platforms, not only Windows.
54 |
55 | Build with (output is stored in `./zig-out/bin`):
56 |
57 | ```sh
58 | zig build -Doptimize=ReleaseFast
59 | ```
60 |
61 | Run debug with:
62 |
63 | ```sh
64 | zig build run -- searchterm
65 | ```
66 |
67 | ## Benchmarks
68 |
69 | Done with [hyperfine](https://github.com/sharkdp/hyperfine). `scoop-search` is on average 350 times faster.
70 |
71 | ```sh
72 | ❯ hyperfine --warmup 1 'scoop-search google' 'scoop search google'
73 | Benchmark 1: scoop-search google
74 | Time (mean ± σ): 60.3 ms ± 3.5 ms [User: 91.2 ms, System: 394.2 ms]
75 | Range (min … max): 55.1 ms … 73.8 ms 49 runs
76 |
77 | Benchmark 2: scoop search google
78 | Time (mean ± σ): 21.275 s ± 2.657 s [User: 9.604 s, System: 11.789 s]
79 | Range (min … max): 19.143 s … 27.035 s 10 runs
80 |
81 | Summary
82 | scoop-search google ran
83 | 352.74 ± 48.49 times faster than scoop search google
84 | ```
85 |
86 | _ran on AMD Ryzen 5 3600 @ 3.6GHz_
87 |
--------------------------------------------------------------------------------
/build.zig:
--------------------------------------------------------------------------------
1 | const std = @import("std");
2 |
3 | // Although this function looks imperative, note that its job is to
4 | // declaratively construct a build graph that will be executed by an external
5 | // runner.
6 | pub fn build(b: *std.Build) void {
7 | // Standard target options allows the person running `zig build` to choose
8 | // what target to build for. Here we do not override the defaults, which
9 | // means any target is allowed, and the default is native. Other options
10 | // for restricting supported target set are available.
11 | const target = b.standardTargetOptions(.{});
12 |
13 | // Standard optimization options allow the person running `zig build` to select
14 | // between Debug, ReleaseSafe, ReleaseFast, and ReleaseSmall. Here we do not
15 | // set a preferred release mode, allowing the user to decide how to optimize.
16 | const optimize = b.standardOptimizeOption(.{});
17 |
18 | // Add mvzr dependency and create module.
19 | const mvzr_dep = b.dependency("mvzr", .{ .target = target, .optimize = optimize });
20 | const mvzr_module = mvzr_dep.module("mvzr");
21 |
22 | const exe = b.addExecutable(.{
23 | .name = "scoop-search",
24 | // In this case the main source file is merely a path, however, in more
25 | // complicated build scripts, this could be a generated file.
26 | .root_source_file = b.path("src/main.zig"),
27 | .target = target,
28 | .optimize = optimize,
29 | });
30 | exe.root_module.addImport("mvzr", mvzr_module);
31 |
32 | // This declares intent for the executable to be installed into the
33 | // standard location when the user invokes the "install" step (the default
34 | // step when running `zig build`).
35 | b.installArtifact(exe);
36 |
37 | // This *creates* a Run step in the build graph, to be executed when another
38 | // step is evaluated that depends on it. The next line below will establish
39 | // such a dependency.
40 | const run_cmd = b.addRunArtifact(exe);
41 |
42 | // By making the run step depend on the install step, it will be run from the
43 | // installation directory rather than directly from within the cache directory.
44 | // This is not necessary, however, if the application depends on other installed
45 | // files, this ensures they will be present and in the expected location.
46 | run_cmd.step.dependOn(b.getInstallStep());
47 |
48 | // This allows the user to pass arguments to the application in the build
49 | // command itself, like this: `zig build run -- arg1 arg2 etc`
50 | if (b.args) |args| {
51 | run_cmd.addArgs(args);
52 | }
53 |
54 | // This creates a build step. It will be visible in the `zig build --help` menu,
55 | // and can be selected like this: `zig build run`
56 | // This will evaluate the `run` step rather than the default, which is "install".
57 | const run_step = b.step("run", "Run the app");
58 | run_step.dependOn(&run_cmd.step);
59 |
60 | // Creates a step for unit testing. This only builds the test executable
61 | // but does not run it.
62 | const unit_tests = b.addTest(.{
63 | .root_source_file = b.path("src/main.zig"),
64 | .target = target,
65 | .optimize = optimize,
66 | });
67 | unit_tests.root_module.addImport("mvzr", mvzr_module);
68 |
69 | const run_unit_tests = b.addRunArtifact(unit_tests);
70 |
71 | // Similar to creating the run step earlier, this exposes a `test` step to
72 | // the `zig build --help` menu, providing a way for the user to request
73 | // running the unit tests.
74 | const test_step = b.step("test", "Run unit tests");
75 | test_step.dependOn(&run_unit_tests.step);
76 | }
77 |
--------------------------------------------------------------------------------
/build.zig.zon:
--------------------------------------------------------------------------------
1 | .{
2 | // This is the default name used by packages depending on this one. For
3 | // example, when a user runs `zig fetch --save `, this field is used
4 | // as the key in the `dependencies` table. Although the user can choose a
5 | // different name, most users will stick with this provided value.
6 | //
7 | // It is redundant to include "zig" in this name because it is already
8 | // within the Zig package namespace.
9 | .name = .scoop_search,
10 |
11 | // This is a [Semantic Version](https://semver.org/).
12 | // In a future version of Zig it will be used for package deduplication.
13 | .version = "0.0.0",
14 |
15 | // This field is optional.
16 | // This is currently advisory only; Zig does not yet do anything
17 | // with this value.
18 | .minimum_zig_version = "0.14.0",
19 | .fingerprint = 0x3efe0610ef868eea,
20 |
21 | // This field is optional.
22 | // Each dependency must either provide a `url` and `hash`, or a `path`.
23 | // `zig build --fetch` can be used to fetch all dependencies of a package, recursively.
24 | // Once all dependencies are fetched, `zig build` no longer requires
25 | // internet connectivity.
26 | .dependencies = .{
27 | .mvzr = .{
28 | .url = "https://github.com/mnemnion/mvzr/archive/refs/tags/v0.3.2.tar.gz",
29 | .hash = "122084c73b4208fdfb02ee2c839e8e204d4b1d93421fecbf1463df96bb4e8a776491",
30 | },
31 | },
32 | .paths = .{
33 | "build.zig",
34 | "build.zig.zon",
35 | "src",
36 | },
37 | }
38 |
--------------------------------------------------------------------------------
/src/args.zig:
--------------------------------------------------------------------------------
1 | const std = @import("std");
2 |
3 | pub const poshHook =
4 | \\function scoop { if ($args[0] -eq "search") { scoop-search.exe @($args | Select-Object -Skip 1) } else { scoop.ps1 @args } }
5 | ;
6 |
7 | pub const ParsedArgs = struct {
8 | query: ?[]const u8,
9 | hook: bool,
10 | allocator: std.mem.Allocator,
11 |
12 | pub fn parse(allocator: std.mem.Allocator) !@This() {
13 | const args = try std.process.argsAlloc(allocator);
14 | defer std.process.argsFree(allocator, args);
15 |
16 | var hook = false;
17 | var query: ?[]const u8 = null;
18 |
19 | if (args.len == 1) {
20 | // pass
21 | } else if (std.mem.eql(u8, args[1], "--hook")) {
22 | hook = true;
23 | } else {
24 | query = try allocator.dupe(u8, args[1]);
25 | }
26 |
27 | return .{ .query = query, .hook = hook, .allocator = allocator };
28 | }
29 |
30 | pub fn deinit(self: *@This()) void {
31 | if (self.query) |q| self.allocator.free(q);
32 | }
33 | };
34 |
--------------------------------------------------------------------------------
/src/env.zig:
--------------------------------------------------------------------------------
1 | const std = @import("std");
2 | const utils = @import("utils.zig");
3 | const testing = std.testing;
4 | const DebugLogger = utils.DebugLogger;
5 |
6 | /// Checks if scoop-search should run in verbose mode.
7 | pub fn isVerbose() bool {
8 | return std.process.hasEnvVarConstant("SCOOP_SEARCH_VERBOSE");
9 | }
10 |
11 | /// Gets the home directory of the current user.
12 | fn homeDirOwned(allocator: std.mem.Allocator, debug: DebugLogger) !?[]const u8 {
13 | const userProfile = std.process.getEnvVarOwned(allocator, "USERPROFILE");
14 | try debug.log("env:USERPROFILE={s}\n", .{userProfile catch ""});
15 |
16 | const dir = userProfile catch |err| switch (err) {
17 | error.EnvironmentVariableNotFound => return null,
18 | else => |e| return e,
19 | };
20 |
21 | if (dir.len == 0) {
22 | allocator.free(dir);
23 | return null;
24 | } else {
25 | return dir;
26 | }
27 | }
28 |
29 | /// Path to the scoop config file.
30 | fn scoopConfigFileOwned(allocator: std.mem.Allocator, homeDir: ?[]const u8, debug: DebugLogger) ![]const u8 {
31 | const xdgConfigHome = std.process.getEnvVarOwned(allocator, "XDG_CONFIG_HOME");
32 | try debug.log("env:XDG_CONFIG_HOME={s}\n", .{xdgConfigHome catch ""});
33 |
34 | const systemConfig = xdgConfigHome catch |err| switch (err) {
35 | error.EnvironmentVariableNotFound => if (homeDir) |dir|
36 | try utils.concatOwned(allocator, dir, "/.config")
37 | else
38 | return error.MissingHomeDir,
39 | else => |e| return e,
40 | };
41 | defer allocator.free(systemConfig);
42 |
43 | return try utils.concatOwned(allocator, systemConfig, "/scoop/config.json");
44 | }
45 |
46 | /// Returns the path to the root of scoop. Logic follows Scoop's logic for resolving the home directory.
47 | pub fn scoopHomeOwned(allocator: std.mem.Allocator, debug: DebugLogger) ![]const u8 {
48 | const scoop = std.process.getEnvVarOwned(allocator, "SCOOP");
49 | try debug.log("env:SCOOP={s}\n", .{scoop catch ""});
50 |
51 | return scoop catch |err| switch (err) {
52 | error.EnvironmentVariableNotFound => {
53 | const homeDir = try homeDirOwned(allocator, debug);
54 | defer {
55 | if (homeDir) |d| allocator.free(d);
56 | }
57 | const scoopConfigPath = try scoopConfigFileOwned(allocator, homeDir, debug);
58 | defer allocator.free(scoopConfigPath);
59 | try debug.log("Scoop config file path: {s}\n", .{scoopConfigPath});
60 |
61 | if (std.fs.openFileAbsolute(scoopConfigPath, .{}) catch null) |configFile| {
62 | defer configFile.close();
63 |
64 | if (utils.readFileOwned(allocator, configFile) catch null) |config| {
65 | defer allocator.free(config);
66 | try debug.log("Scoop config file contents: {s}\n", .{config});
67 |
68 | const parsed = try std.json.parseFromSlice(struct { root_path: ?[]const u8 = null }, allocator, config, .{ .ignore_unknown_fields = true });
69 | defer parsed.deinit();
70 | const rootPath = parsed.value.root_path orelse "";
71 |
72 | if (rootPath.len != 0) {
73 | return allocator.dupe(u8, rootPath);
74 | }
75 | }
76 | } else {
77 | try debug.log("Scoop config file does not exist\n", .{});
78 | }
79 |
80 | // installing with default directory doesn't have `SCOOP`
81 | // and `root_path` either
82 | return if (homeDir) |dir|
83 | try utils.concatOwned(allocator, dir, "/scoop")
84 | else
85 | return error.MissingHomeDir;
86 | },
87 | else => |e| return e,
88 | };
89 | }
90 |
--------------------------------------------------------------------------------
/src/main.zig:
--------------------------------------------------------------------------------
1 | const std = @import("std");
2 | const poshHook = @import("args.zig").poshHook;
3 | const ParsedArgs = @import("args.zig").ParsedArgs;
4 | const env = @import("env.zig");
5 | const utils = @import("utils.zig");
6 | const search = @import("search.zig");
7 | const ThreadPool = @import("thread_pool.zig").ThreadPool;
8 | const mvzr = @import("mvzr");
9 |
10 | /// Stores results of a search in a single bucket.
11 | const SearchResult = struct {
12 | bucketName: []const u8,
13 | result: search.SearchResult,
14 | allocator: std.mem.Allocator,
15 |
16 | fn init(allocator: std.mem.Allocator, bucketName: []const u8, result: search.SearchResult) !@This() {
17 | return .{
18 | .bucketName = try allocator.dupe(u8, bucketName),
19 | .result = result,
20 | .allocator = allocator,
21 | };
22 | }
23 |
24 | fn deinit(self: *@This()) void {
25 | self.allocator.free(self.bucketName);
26 | self.result.deinit();
27 | }
28 | };
29 |
30 | pub fn main() !void {
31 | var heap = utils.HeapAllocator.init();
32 | defer heap.deinit();
33 | const allocator = heap.allocator();
34 |
35 | var args = try ParsedArgs.parse(allocator);
36 | defer args.deinit();
37 |
38 | // print posh hook and exit if requested
39 | if (args.hook) {
40 | try std.io.getStdOut().writer().print("{s}\n", .{poshHook});
41 | std.process.exit(0);
42 | }
43 |
44 | const debug = utils.DebugLogger.init(env.isVerbose());
45 | try debug.log("Commandline arguments: {}\n", .{args});
46 |
47 | const query = try std.ascii.allocLowerString(allocator, args.query orelse "");
48 | defer allocator.free(query);
49 |
50 | const regexQuery = mvzr.compile(query) orelse
51 | return std.io.getStdErr().writer().print("Invalid regular expression: parsing \"{s}\".", .{query});
52 |
53 | const scoopHome = env.scoopHomeOwned(allocator, debug) catch |err| switch (err) {
54 | error.MissingHomeDir => {
55 | return std.io.getStdErr().writer().print("Could not establish scoop home directory. USERPROFILE environment variable is not defined.\n", .{});
56 | },
57 | else => |e| return e,
58 | };
59 | defer allocator.free(scoopHome);
60 | try debug.log("Scoop home: {s}\n", .{scoopHome});
61 |
62 | // get buckets path
63 | const bucketsPath = try utils.concatOwned(allocator, scoopHome, "/buckets");
64 | defer allocator.free(bucketsPath);
65 |
66 | var bucketsDir = std.fs.openDirAbsolute(bucketsPath, .{ .iterate = true }) catch
67 | return std.io.getStdErr().writer().print("Could not open the buckets directory: {s}.\n", .{bucketsPath});
68 | defer bucketsDir.close();
69 |
70 | // search each bucket one by one
71 | var results = std.ArrayList(SearchResult).init(allocator);
72 | defer {
73 | for (results.items) |*e| e.deinit();
74 | results.deinit();
75 | }
76 | var iter = bucketsDir.iterate();
77 | while (try iter.next()) |f| {
78 | if (f.kind != .directory) {
79 | continue;
80 | }
81 |
82 | const bucketBase = try std.mem.concat(allocator, u8, &[_][]const u8{ bucketsPath, "/", f.name });
83 | defer allocator.free(bucketBase);
84 | try debug.log("Found bucket: {s}\n", .{bucketBase});
85 |
86 | const result = search.searchBucket(allocator, regexQuery, bucketBase, debug) catch {
87 | try std.io.getStdErr().writer().print("Failed to search through the bucket: {s}.\n", .{f.name});
88 | continue;
89 | };
90 | try debug.log("Found {} matches\n", .{result.matches.items.len});
91 |
92 | try results.append(try SearchResult.init(
93 | allocator,
94 | f.name,
95 | result,
96 | ));
97 | }
98 |
99 | try debug.log("Done searching\n", .{});
100 |
101 | const hasMatches = try printResults(&results);
102 | if (!hasMatches)
103 | std.process.exit(1);
104 | }
105 |
106 | /// Returns whether there were any matches.
107 | fn printResults(results: *std.ArrayList(SearchResult)) !bool {
108 | const SearchResultBucketSort = struct {
109 | fn lessThan(context: void, lhs: SearchResult, rhs: SearchResult) bool {
110 | _ = context;
111 | return std.mem.order(u8, lhs.bucketName, rhs.bucketName).compare(.lt);
112 | }
113 | };
114 | // sort results by bucket name
115 | std.mem.sort(SearchResult, results.items, {}, SearchResultBucketSort.lessThan);
116 |
117 | var hasMatches = false;
118 |
119 | // get a buffered writer for stdout
120 | const stdout = std.io.getStdOut();
121 | var bw = std.io.BufferedWriter(8 * 1024, std.fs.File.Writer){ .unbuffered_writer = stdout.writer() };
122 | defer stdout.close();
123 |
124 | for (results.items) |*result| {
125 | if (result.result.matches.items.len == 0) {
126 | continue;
127 | }
128 | hasMatches = true;
129 |
130 | _ = try bw.write("'");
131 | _ = try bw.write(result.bucketName);
132 | _ = try bw.write("' bucket:\n");
133 |
134 | for (result.result.matches.items) |match| {
135 | _ = try bw.write(" ");
136 | _ = try bw.write(match.name);
137 | _ = try bw.write(" (");
138 | _ = try bw.write(match.version);
139 | _ = try bw.write(")");
140 | if (match.bins.items.len != 0) {
141 | _ = try bw.write(" --> includes '");
142 | _ = try bw.write(match.bins.items[0]);
143 | _ = try bw.write("'");
144 | }
145 | _ = try bw.write("\n");
146 | }
147 | _ = try bw.write("\n");
148 | }
149 |
150 | if (!hasMatches) {
151 | const colorConfig = std.io.tty.detectConfig(stdout);
152 |
153 | try colorConfig.setColor(bw.writer(), std.io.tty.Color.yellow);
154 | _ = try bw.write("WARN No matches found.\n");
155 | try colorConfig.setColor(bw.writer(), std.io.tty.Color.reset);
156 | }
157 |
158 | try bw.flush();
159 | return hasMatches;
160 | }
161 |
--------------------------------------------------------------------------------
/src/search.zig:
--------------------------------------------------------------------------------
1 | const std = @import("std");
2 |
3 | const mvzr = @import("mvzr");
4 |
5 | const utils = @import("utils.zig");
6 | const Box = utils.Box;
7 | const DebugLogger = utils.DebugLogger;
8 |
9 | /// State associated with a worker thread. Stores thread local cache and matches. Has its own allocator.
10 | const ThreadPoolState = struct {
11 | matches: std.ArrayList(SearchMatch),
12 | read_buffer: []u8,
13 | allocator: Box(utils.HeapAllocator),
14 |
15 | fn create(allocator: std.mem.Allocator) !@This() {
16 | var allocat = try Box(utils.HeapAllocator).init(allocator, utils.HeapAllocator.init());
17 |
18 | return .{
19 | .matches = std.ArrayList(SearchMatch).init(allocat.ptr.allocator()),
20 | .read_buffer = try allocat.ptr.allocator().alloc(u8, 8 * 1024),
21 | .allocator = allocat,
22 | };
23 | }
24 |
25 | /// Merges many states into one. Takes ownership of states and frees appropriate resources.
26 | pub fn mergeStates(allocator: std.mem.Allocator, states: std.ArrayList(@This())) !SearchResult {
27 | var totalMatches: usize = 0;
28 | for (states.items) |*e| totalMatches += e.matches.items.len;
29 |
30 | var matches = try std.ArrayList(SearchMatch).initCapacity(allocator, totalMatches);
31 | var allocators = try std.ArrayList(Box(utils.HeapAllocator)).initCapacity(allocator, states.items.len);
32 | for (states.items) |*e| {
33 | e.allocator.ptr.allocator().free(e.read_buffer);
34 |
35 | try allocators.append(e.allocator);
36 |
37 | try matches.appendSlice(e.matches.items);
38 | e.matches.deinit();
39 | }
40 |
41 | states.deinit();
42 |
43 | return .{
44 | .matches = matches,
45 | .allocators = allocators,
46 | };
47 | }
48 |
49 | pub fn deinit(self: *@This()) void {
50 | for (self.matches.items) |*e| e.deinit();
51 | self.matches.deinit();
52 |
53 | self.allocator.ptr.allocator().free(self.read_buffer);
54 |
55 | self.allocator.ptr.deinit();
56 | self.allocator.deinit();
57 | }
58 | };
59 |
60 | /// A merged result of a single bucket search.
61 | pub const SearchResult = struct {
62 | matches: std.ArrayList(SearchMatch),
63 | allocators: std.ArrayList(Box(utils.HeapAllocator)),
64 |
65 | fn sortMatches(self: *@This()) void {
66 | const Sort = struct {
67 | fn lessThan(context: void, lhs: SearchMatch, rhs: SearchMatch) bool {
68 | _ = context;
69 | return std.mem.order(u8, lhs.name, rhs.name).compare(.lt);
70 | }
71 | };
72 | // sort results by package name
73 | std.mem.sort(SearchMatch, self.matches.items, {}, Sort.lessThan);
74 | }
75 |
76 | pub fn deinit(self: *@This()) void {
77 | for (self.matches.items) |*e| e.deinit();
78 | self.matches.deinit();
79 |
80 | for (self.allocators.items) |*e| {
81 | e.ptr.deinit();
82 | e.deinit();
83 | }
84 | self.allocators.deinit();
85 | }
86 | };
87 | const ThreadPool = @import("thread_pool.zig").ThreadPool(ThreadPoolState);
88 |
89 | /// A single match of a package inside of the current bucket.
90 | pub const SearchMatch = struct {
91 | name: []const u8,
92 | version: []const u8,
93 | bins: std.ArrayList([]const u8),
94 | allocator: std.mem.Allocator,
95 |
96 | fn init(allocator: std.mem.Allocator, name: []const u8, version: []const u8, bins: std.ArrayList([]const u8)) !@This() {
97 | return .{
98 | .name = try allocator.dupe(u8, name),
99 | .version = try allocator.dupe(u8, version),
100 | .bins = blk: {
101 | var dupedBins = try std.ArrayList([]const u8).initCapacity(allocator, bins.items.len);
102 | for (bins.items) |bin| {
103 | try dupedBins.append(try allocator.dupe(u8, bin));
104 | }
105 | break :blk dupedBins;
106 | },
107 | .allocator = allocator,
108 | };
109 | }
110 |
111 | pub fn deinit(self: *@This()) void {
112 | self.allocator.free(self.name);
113 | self.allocator.free(self.version);
114 | for (self.bins.items) |bin| {
115 | self.allocator.free(bin);
116 | }
117 | self.bins.deinit();
118 | }
119 | };
120 |
121 | /// Returns the directory where manifests are stored for the given bucket.
122 | fn getPackagesDir(allocator: std.mem.Allocator, bucketBase: []const u8) !std.fs.Dir {
123 | // check if $bucketName/bucket exists, if not use $bucketName
124 | const packagesPath = try utils.concatOwned(allocator, bucketBase, "/bucket");
125 | defer allocator.free(packagesPath);
126 |
127 | const packages = std.fs.openDirAbsolute(packagesPath, .{ .iterate = true }) catch
128 | // fallback to $bucketName
129 | try std.fs.openDirAbsolute(bucketBase, .{ .iterate = true });
130 |
131 | return packages;
132 | }
133 |
134 | pub fn searchBucket(allocator: std.mem.Allocator, query: mvzr.Regex, bucketBase: []const u8, debug: DebugLogger) !SearchResult {
135 | var tp: ThreadPool = undefined;
136 | try tp.init(.{ .allocator = allocator }, ThreadPoolState.create);
137 | try debug.log("Worker count: {}\n", .{tp.threads.len});
138 |
139 | var packages = try getPackagesDir(allocator, bucketBase);
140 | defer packages.close();
141 |
142 | var names = std.ArrayList([]const u8).init(allocator);
143 | defer {
144 | for (names.items) |e| allocator.free(e);
145 | names.deinit();
146 | }
147 |
148 | var walker = try packages.walk(allocator);
149 | defer walker.deinit();
150 | while (try walker.next()) |f| {
151 | if (f.kind != .file) {
152 | continue;
153 | }
154 |
155 | try names.append(try allocator.dupe(u8, f.path));
156 |
157 | try tp.spawn(matchPackage, .{ packages, query, names.getLast() });
158 | }
159 |
160 | const states = tp.deinit();
161 |
162 | var result = try ThreadPoolState.mergeStates(allocator, states);
163 | result.sortMatches();
164 |
165 | return result;
166 | }
167 |
168 | /// If the given binary name matches the query, return it.
169 | fn checkBin(allocator: std.mem.Allocator, bin: []const u8, query: mvzr.Regex) !?[]const u8 {
170 | const against = utils.basename(bin);
171 | const lowerBinStem = try std.ascii.allocLowerString(allocator, against.withoutExt);
172 | defer allocator.free(lowerBinStem);
173 |
174 | return if (query.isMatch(lowerBinStem)) against.withExt else null;
175 | }
176 |
177 | fn matchPackage(packagesDir: std.fs.Dir, query: mvzr.Regex, manifestName: []const u8, state: *ThreadPoolState) void {
178 | // ignore failed match
179 | matchPackageAux(packagesDir, query, manifestName, state) catch return;
180 | }
181 |
182 | /// A worker function for checking if a given manifest matches the query.
183 | fn matchPackageAux(packagesDir: std.fs.Dir, query: mvzr.Regex, manifestPath: []const u8, state: *ThreadPoolState) !void {
184 | const allocator = state.allocator.ptr.allocator();
185 |
186 | const extension = comptime ".json";
187 | if (!std.mem.endsWith(u8, manifestPath, extension)) {
188 | return;
189 | }
190 | const stem = utils.basename(manifestPath).withoutExt;
191 |
192 | const manifest = try packagesDir.openFile(manifestPath, .{});
193 | defer manifest.close();
194 | const content = try utils.readFileRealloc(allocator, manifest, &state.read_buffer);
195 |
196 | const Manifest = struct {
197 | version: ?[]const u8 = null,
198 | bin: ?std.json.Value = null, // can be: null, string, [](string | []string)
199 | };
200 |
201 | // skip invalid manifests
202 | const parsed = std.json.parseFromSlice(Manifest, allocator, content, .{ .ignore_unknown_fields = true }) catch return;
203 | defer parsed.deinit();
204 | const version = parsed.value.version orelse "";
205 |
206 | const lowerStem = try std.ascii.allocLowerString(allocator, stem);
207 | defer allocator.free(lowerStem);
208 |
209 | var matchedBins = std.ArrayList([]const u8).init(allocator);
210 | defer matchedBins.deinit();
211 |
212 | // does the package name match?
213 | if (query.isMatch(lowerStem)) {
214 | try state.matches.append(try SearchMatch.init(allocator, stem, version, matchedBins));
215 | } else {
216 | // the name did not match, lets see if any binary files do
217 | switch (parsed.value.bin orelse .null) {
218 | .string => |bin| {
219 | if (try checkBin(allocator, bin, query)) |matchedBin| {
220 | try matchedBins.append(matchedBin);
221 | }
222 | },
223 | .array => |bins| for (bins.items) |e|
224 | switch (e) {
225 | .string => |bin| if (try checkBin(allocator, bin, query)) |matchedBin| {
226 | try matchedBins.append(matchedBin);
227 | },
228 | .array => |args| {
229 | // check only first two (exe, alias), the rest are command flags
230 | if (args.items.len > 0) {
231 | switch (args.items[0]) {
232 | .string => |bin| if (try checkBin(allocator, bin, query)) |matchedBin| {
233 | try matchedBins.append(matchedBin);
234 | },
235 | else => {},
236 | }
237 | }
238 | if (args.items.len > 1) {
239 | switch (args.items[1]) {
240 | .string => |bin| if (try checkBin(allocator, bin, query)) |matchedBin| {
241 | try matchedBins.append(matchedBin);
242 | },
243 | else => {},
244 | }
245 | }
246 | },
247 | else => continue,
248 | },
249 | else => return,
250 | }
251 |
252 | if (matchedBins.items.len != 0) {
253 | try state.matches.append(try SearchMatch.init(allocator, stem, version, matchedBins));
254 | }
255 | }
256 | }
257 |
--------------------------------------------------------------------------------
/src/thread_pool.zig:
--------------------------------------------------------------------------------
1 | /// Fork of std.Thread.Pool with a persisted state for each thread.
2 | const std = @import("std");
3 | const builtin = @import("builtin");
4 |
5 | pub fn ThreadPool(comptime T: type) type {
6 | return struct {
7 | const Self = @This();
8 |
9 | mutex: std.Thread.Mutex = .{},
10 | cond: std.Thread.Condition = .{},
11 | run_queue: RunQueue = .{},
12 | is_running: bool = true,
13 | allocator: std.mem.Allocator,
14 | threads: []std.Thread,
15 | states: std.ArrayList(T),
16 |
17 | const RunQueue = std.SinglyLinkedList(Runnable);
18 | const Runnable = struct {
19 | runFn: RunProto,
20 | };
21 |
22 | const RunProto = *const fn (*Runnable, *T) void;
23 |
24 | pub const Options = struct {
25 | allocator: std.mem.Allocator,
26 | n_jobs: ?u32 = null,
27 | };
28 |
29 | pub fn init(pool: *Self, options: Options, comptime create_state: *const fn (std.mem.Allocator) std.mem.Allocator.Error!T) !void {
30 | const allocator = options.allocator;
31 |
32 | const thread_count = options.n_jobs orelse @max(1, std.Thread.getCpuCount() catch 1);
33 |
34 | pool.* = .{
35 | .allocator = allocator,
36 | .threads = &[_]std.Thread{},
37 | .states = try std.ArrayList(T).initCapacity(allocator, thread_count),
38 | };
39 |
40 | if (builtin.single_threaded) {
41 | return;
42 | }
43 |
44 | pool.threads = try allocator.alloc(std.Thread, thread_count);
45 | errdefer allocator.free(pool.threads);
46 |
47 | // kill and join any threads we spawned previously on error.
48 | var spawned: usize = 0;
49 | errdefer {
50 | pool.join(spawned);
51 | for (pool.states.items) |*e| e.deinit();
52 | pool.states.deinit();
53 | }
54 |
55 | for (pool.threads) |*thread| {
56 | try pool.states.append(try create_state(allocator));
57 | thread.* = try std.Thread.spawn(.{}, worker, .{
58 | pool,
59 | // pointer shall not move after `append` above; we allocated needed capacity at the start
60 | &pool.states.items[pool.states.items.len - 1],
61 | });
62 | spawned += 1;
63 | }
64 | }
65 |
66 | pub fn deinit(pool: *Self) std.ArrayList(T) {
67 | pool.join(pool.threads.len); // kill and join all threads.
68 | const states = pool.states;
69 | pool.* = undefined;
70 | return states;
71 | }
72 |
73 | fn join(pool: *Self, spawned: usize) void {
74 | if (builtin.single_threaded) {
75 | return;
76 | }
77 |
78 | {
79 | pool.mutex.lock();
80 | defer pool.mutex.unlock();
81 |
82 | // ensure future worker threads exit the dequeue loop
83 | pool.is_running = false;
84 | }
85 |
86 | // wake up any sleeping threads (this can be done outside the mutex)
87 | // then wait for all the threads we know are spawned to complete.
88 | pool.cond.broadcast();
89 | for (pool.threads[0..spawned]) |*thread| {
90 | thread.join();
91 | }
92 |
93 | pool.allocator.free(pool.threads);
94 | }
95 |
96 | pub fn spawn(pool: *Self, comptime func: anytype, args: anytype) !void {
97 | if (builtin.single_threaded) {
98 | @call(.auto, func, args);
99 | return;
100 | }
101 |
102 | const Args = @TypeOf(args);
103 | const Closure = struct {
104 | arguments: Args,
105 | pool: *Self,
106 | run_node: RunQueue.Node = .{ .data = .{ .runFn = runFn } },
107 |
108 | fn runFn(runnable: *Runnable, state: *T) void {
109 | const run_node: *RunQueue.Node = @fieldParentPtr("data", runnable);
110 | const closure: *@This() = @fieldParentPtr("run_node", run_node);
111 | @call(.auto, func, closure.arguments ++ .{state});
112 |
113 | // The thread pool's allocator is protected by the mutex.
114 | const mutex = &closure.pool.mutex;
115 | mutex.lock();
116 | defer mutex.unlock();
117 |
118 | closure.pool.allocator.destroy(closure);
119 | }
120 | };
121 |
122 | {
123 | pool.mutex.lock();
124 | defer pool.mutex.unlock();
125 |
126 | const closure = try pool.allocator.create(Closure);
127 | closure.* = .{
128 | .arguments = args,
129 | .pool = pool,
130 | };
131 |
132 | pool.run_queue.prepend(&closure.run_node);
133 | }
134 |
135 | // Notify waiting threads outside the lock to try and keep the critical section small.
136 | pool.cond.signal();
137 | }
138 |
139 | fn worker(pool: *Self, state: *T) void {
140 | pool.mutex.lock();
141 | defer pool.mutex.unlock();
142 |
143 | while (true) {
144 | while (pool.run_queue.popFirst()) |run_node| {
145 | // Temporarily unlock the mutex in order to execute the run_node
146 | pool.mutex.unlock();
147 | defer pool.mutex.lock();
148 |
149 | const runFn = run_node.data.runFn;
150 | runFn(&run_node.data, state);
151 | }
152 |
153 | // Stop executing instead of waiting if the thread pool is no longer running.
154 | if (pool.is_running) {
155 | pool.cond.wait(&pool.mutex);
156 | } else {
157 | break;
158 | }
159 | }
160 | }
161 | };
162 | }
163 |
--------------------------------------------------------------------------------
/src/utils.zig:
--------------------------------------------------------------------------------
1 | const std = @import("std");
2 | const assert = std.debug.assert;
3 | const builtin = @import("builtin");
4 |
5 | /// A heap allocator optimized for each platform/compilation mode.
6 | pub const HeapAllocator = struct {
7 | const is_windows = builtin.target.os.tag == .windows;
8 |
9 | backing_allocator: if (builtin.mode == .Debug)
10 | std.heap.GeneralPurposeAllocator(.{})
11 | else if (is_windows)
12 | std.mem.Allocator
13 | else if (builtin.link_libc)
14 | void
15 | else
16 | @compileError("When not running in debug mode, you must use windows or link to libc as zig does not provide a fast, libc-less, general purpose allocator (yet)"),
17 |
18 | pub fn init() @This() {
19 | return .{ .backing_allocator = if (builtin.mode == .Debug)
20 | std.heap.GeneralPurposeAllocator(.{}){}
21 | else if (is_windows)
22 | std.heap.smp_allocator
23 | else if (builtin.link_libc) {} };
24 | }
25 |
26 | pub fn allocator(self: *@This()) std.mem.Allocator {
27 | if (builtin.mode == .Debug) {
28 | return self.backing_allocator.allocator();
29 | } else if (is_windows) {
30 | return self.backing_allocator;
31 | } else if (builtin.link_libc) {
32 | return std.heap.c_allocator;
33 | }
34 | }
35 |
36 | pub fn deinit(self: *@This()) void {
37 | if (builtin.mode == .Debug) {
38 | assert(self.backing_allocator.deinit() == .ok);
39 | }
40 | }
41 | };
42 |
43 | /// Concatenates two slices into a newly allocated buffer. Returns an owned reference slice into the contents.
44 | pub fn concatOwned(allocator: std.mem.Allocator, a: []const u8, b: []const u8) ![]const u8 {
45 | const result = try allocator.alloc(u8, a.len + b.len);
46 | @memcpy(result[0..a.len], a);
47 | @memcpy(result[a.len..], b);
48 | return result;
49 | }
50 |
51 | /// Reads a file into a newly allocated buffer. Returns an owned reference slice into the contents.
52 | pub fn readFileOwned(allocator: std.mem.Allocator, file: std.fs.File) ![]const u8 {
53 | const stat = try file.stat();
54 | const buffer = try allocator.alloc(u8, @as(usize, stat.size));
55 |
56 | assert(try file.readAll(buffer) == stat.size);
57 |
58 | return buffer;
59 | }
60 |
61 | /// Reads a file into a provided buffer. Returns a reference slice into the contents. Buffer will be reallocated if more space is needed.
62 | pub fn readFileRealloc(allocator: std.mem.Allocator, file: std.fs.File, buffer: *[]u8) ![]const u8 {
63 | var index: usize = 0;
64 | assert(buffer.len > 0);
65 |
66 | while (true) {
67 | // we need more space for the buffer, double it
68 | if (index == buffer.len) {
69 | buffer.* = try allocator.realloc(buffer.*, buffer.len * 2);
70 | }
71 | const amt = try file.read(buffer.*[index..]);
72 | if (amt == 0) break;
73 | index += amt;
74 | }
75 |
76 | return buffer.*[0..index];
77 | }
78 |
79 | /// Returns the basename of a path with and without the extension.
80 | pub fn basename(path: []const u8) struct { withExt: []const u8, withoutExt: []const u8 } {
81 | const base = std.fs.path.basename(path);
82 | const ext = std.fs.path.extension(base);
83 | return .{ .withExt = base, .withoutExt = base[0..(base.len - ext.len)] };
84 | }
85 |
86 | /// An owned pointer allocated using an allocator. ptr address will not move.
87 | /// Similar to Rust's Box.
88 | pub fn Box(comptime T: type) type {
89 | return struct {
90 | const Self = @This();
91 |
92 | ptr: *T,
93 | allocator: std.mem.Allocator,
94 |
95 | /// Allocates a new Box using the provided allocator.
96 | pub fn init(allocator: std.mem.Allocator, value: T) !Self {
97 | const ptr = try allocator.create(T);
98 | ptr.* = value;
99 | return .{ .ptr = ptr, .allocator = allocator };
100 | }
101 |
102 | pub fn deinit(self: *Self) void {
103 | self.allocator.destroy(self.ptr);
104 | }
105 | };
106 | }
107 |
108 | pub const DebugLogger = struct {
109 | writer: ?std.fs.File.Writer,
110 |
111 | pub fn init(enabled: bool) @This() {
112 | return .{ .writer = if (enabled) std.io.getStdErr().writer() else null };
113 | }
114 |
115 | pub inline fn log(self: @This(), comptime format: []const u8, args: anytype) !void {
116 | if (self.writer) |out| try out.print(format, args);
117 | }
118 | };
119 |
--------------------------------------------------------------------------------