childrenMap = new HashMap<>();
96 | for (Directory child : tree.getChildrenList()) {
97 | childrenMap.put(digestUtil.compute(child), child);
98 | }
99 | downloadDirectory(downloadPath, tree.getRoot(), childrenMap);
100 | }
101 |
102 | /**
103 | * Download a directory recursively. The directory is represented by a {@link Directory} protobuf
104 | * message, and the descendant directories are in {@code childrenMap}, accessible through their
105 | * digest.
106 | *
107 | * Failure can result in partially downloaded directory contents.
108 | *
109 | * @throws IOException in case of a cache miss or if the remote cache is unavailable.
110 | */
111 | private void downloadDirectory(Path path, Directory dir, Map childrenMap)
112 | throws IOException {
113 | // Ensure that the directory is created here even though the directory might be empty
114 | Files.createDirectories(path);
115 | for (FileNode child : dir.getFilesList()) {
116 | Path childPath = path.resolve(child.getName());
117 | try {
118 | downloadFile(childPath, child.getDigest(), child.getIsExecutable(), null);
119 | } catch (IOException e) {
120 | throw new IOException(String.format("Failed to download file %s", child.getName()), e);
121 | }
122 | }
123 | for (DirectoryNode child : dir.getDirectoriesList()) {
124 | Path childPath = path.resolve(child.getName());
125 | Digest childDigest = child.getDigest();
126 | Directory childDir = childrenMap.get(childDigest);
127 | if (childDir == null) {
128 | throw new IOException(
129 | "could not find subdirectory "
130 | + child.getName()
131 | + " of directory "
132 | + path
133 | + " for download: digest "
134 | + childDigest
135 | + "not found");
136 | }
137 | downloadDirectory(childPath, childDir, childrenMap);
138 | }
139 | }
140 |
141 | /**
142 | * Download the full contents of a OutputDirectory to a local path.
143 | *
144 | * @param dir The OutputDirectory to download.
145 | * @param path The path to download the directory to.
146 | * @throws IOException
147 | */
148 | public void downloadOutputDirectory(OutputDirectory dir, Path path) throws IOException {
149 | Tree tree;
150 | try {
151 | tree = Tree.parseFrom(downloadBlob(dir.getTreeDigest()));
152 | } catch (IOException e) {
153 | throw new IOException("Could not obtain tree for OutputDirectory.", e);
154 | }
155 | Map childrenMap = new HashMap<>();
156 | for (Directory child : tree.getChildrenList()) {
157 | childrenMap.put(digestUtil.compute(child), child);
158 | }
159 | downloadDirectory(path, tree.getRoot(), childrenMap);
160 | }
161 |
162 | /** Sets owner executable permission depending on isExecutable. */
163 | private void setExecutable(Path path, boolean isExecutable) throws IOException {
164 | Set originalPerms = Files.getPosixFilePermissions(path); // Immutable.
165 | Set perms = EnumSet.copyOf(originalPerms);
166 | if (!isExecutable) {
167 | perms.remove(PosixFilePermission.OWNER_EXECUTE);
168 | } else {
169 | perms.add(PosixFilePermission.OWNER_EXECUTE);
170 | }
171 | Files.setPosixFilePermissions(path, perms);
172 | }
173 |
174 | /**
175 | * Download a file (that is not a directory). If the {@code content} is not given, the content is
176 | * fetched from the digest.
177 | */
178 | protected void downloadFile(
179 | Path path, Digest digest, boolean isExecutable, @Nullable ByteString content)
180 | throws IOException {
181 | Files.createDirectories(path.getParent());
182 | if (digest.getSizeBytes() == 0) {
183 | // Handle empty file locally.
184 | Files.createFile(path);
185 | } else {
186 | if (content != null && !content.isEmpty()) {
187 | try (OutputStream stream = Files.newOutputStream(path)) {
188 | content.writeTo(stream);
189 | }
190 | } else {
191 | downloadBlob(digest, path);
192 | Digest receivedDigest = digestUtil.compute(path);
193 | if (!receivedDigest.equals(digest)) {
194 | throw new IOException("Digest does not match " + receivedDigest + " != " + digest);
195 | }
196 | }
197 | }
198 | // setExecutable doesn't work on Windows, nor is it required.
199 | if (!System.getProperty("os.name").startsWith("Windows")) {
200 | setExecutable(path, isExecutable);
201 | }
202 | }
203 |
204 | /**
205 | * Download a remote blob to a local destination.
206 | *
207 | * @param digest The digest of the remote blob.
208 | * @param dest The path to the local file.
209 | * @throws IOException if download failed.
210 | */
211 | protected abstract void downloadBlob(Digest digest, Path dest) throws IOException;
212 |
213 | /**
214 | * Download a remote blob and store it in memory.
215 | *
216 | * @param digest The digest of the remote blob.
217 | * @return The remote blob.
218 | * @throws IOException if download failed.
219 | */
220 | protected abstract byte[] downloadBlob(Digest digest) throws IOException;
221 |
222 | /**
223 | * Download a single blob with the specified digest into an output stream.
224 | *
225 | * @throws CacheNotFoundException in case of a cache miss.
226 | */
227 | public abstract void downloadBlob(Digest digest, OutputStream dest) throws IOException;
228 | }
229 |
--------------------------------------------------------------------------------
/src/main/java/com/google/devtools/build/remote/client/ActionGrouping.java:
--------------------------------------------------------------------------------
1 | package com.google.devtools.build.remote.client;
2 |
3 | import build.bazel.remote.execution.v2.Digest;
4 | import build.bazel.remote.execution.v2.ExecuteResponse;
5 | import com.google.common.annotations.VisibleForTesting;
6 | import com.google.common.collect.Multiset;
7 | import com.google.common.collect.TreeMultiset;
8 | import com.google.devtools.build.lib.remote.logging.RemoteExecutionLog.LogEntry;
9 | import com.google.protobuf.util.Timestamps;
10 | import io.grpc.Status;
11 | import java.io.IOException;
12 | import java.io.PrintWriter;
13 | import java.util.ArrayList;
14 | import java.util.LinkedHashMap;
15 | import java.util.List;
16 | import java.util.Map;
17 | import java.util.stream.Collectors;
18 | import org.json.simple.JSONArray;
19 | import org.json.simple.JSONObject;
20 | import org.json.simple.JSONValue;
21 |
22 | /** A class to handle GRPc log grouped by actions */
23 | final class ActionGrouping {
24 |
25 | @VisibleForTesting
26 | static final String actionDelimiter = "************************************************";
27 |
28 | @VisibleForTesting
29 | static final String entryDelimiter = "------------------------------------------------";
30 |
31 | @VisibleForTesting static final String actionString = "Entries for action with hash '%s'\n";
32 |
33 | @VisibleForTesting
34 | static class ActionDetails {
35 | final Multiset log;
36 | final Digest digest;
37 | final String actionId;
38 | final ExecuteResponse executeResponse;
39 |
40 | private ActionDetails(
41 | Multiset log, Digest digest, String actionId, ExecuteResponse executeResponse) {
42 | this.log = log;
43 | this.digest = digest;
44 | this.actionId = actionId;
45 | this.executeResponse = executeResponse;
46 | }
47 |
48 | Digest getDigest() {
49 | return digest;
50 | }
51 |
52 | public ExecuteResponse getExecuteResponse() {
53 | return executeResponse;
54 | }
55 |
56 | // We will consider an action to be failed if either:
57 | // - we successfully received the execution result but the status is non-zero
58 | // - we successfully received the action result but the exit code is non-zero
59 | boolean isFailed() {
60 | if (executeResponse == null) {
61 | // Action was not successfully completed (either cancelled or RPC error)
62 | // We don't know if it's a failing action.
63 | return false;
64 | }
65 |
66 | if (executeResponse.hasStatus()
67 | && executeResponse.getStatus().getCode() != Status.Code.OK.value()) {
68 | // Errors such as PERMISSION_DENIED or DEADLINE_EXCEEDED
69 | return true;
70 | }
71 |
72 | // Return true if the action was not successful
73 | return executeResponse.hasResult() && executeResponse.getResult().getExitCode() != 0;
74 | }
75 |
76 | Iterable getSortedElements() {
77 | return log;
78 | }
79 |
80 | public static class Builder {
81 | Multiset log;
82 | Digest digest;
83 | String actionId;
84 | ExecuteResponse executeResponse;
85 |
86 | public Builder(String actionId) {
87 | log =
88 | TreeMultiset.create(
89 | (a, b) -> {
90 | int i = Timestamps.compare(a.getStartTime(), b.getStartTime());
91 | if (i != 0) {
92 | return i;
93 | }
94 | // In the improbable case of the same timestamp, ensure the messages do not
95 | // override each other.
96 | return a.hashCode() - b.hashCode();
97 | });
98 | this.actionId = actionId;
99 | }
100 |
101 | void add(LogEntry entry) throws IOException {
102 | log.add(entry);
103 |
104 | Digest d = LogParserUtils.extractDigest(entry);
105 | if (d != null) {
106 | if (digest != null && !d.equals(digest)) {
107 | System.err.println("Warning: conflicting digests: " + d + " and " + digest);
108 | }
109 | if (d != null && !d.getHash().equals(actionId)) {
110 | System.err.println(
111 | "Warning: bad digest: " + d + " doesn't match action Id " + actionId);
112 | }
113 | digest = d;
114 | }
115 |
116 | List r = LogParserUtils.extractExecuteResponse(entry);
117 | if (r.size() > 0) {
118 | if(r.size() > 1) {
119 | System.err.println(
120 | "Warning: unexpected log format: multiple ExecutionResponse for action " + actionId
121 | + " in LogEntry " + entry);
122 | }
123 | if (executeResponse != null && executeResponse.hasResult()) {
124 | System.err.println(
125 | "Warning: unexpected log format: multiple action results for action " + actionId);
126 | }
127 | executeResponse = r.get(r.size() - 1);
128 | }
129 | }
130 |
131 | ActionDetails build() {
132 | return new ActionDetails(log, digest, actionId, executeResponse);
133 | }
134 | }
135 | };
136 |
137 | private final Map actionMap;
138 |
139 | private ActionGrouping(Map actionMap) {
140 | this.actionMap = actionMap;
141 | };
142 |
143 | void printByAction(PrintWriter out) throws IOException {
144 | for (String hash : actionMap.keySet()) {
145 | out.println(actionDelimiter);
146 | out.printf(actionString, hash);
147 | out.println(actionDelimiter);
148 | for (LogEntry entry : actionMap.get(hash).getSortedElements()) {
149 | LogParserUtils.printLogEntry(entry, out);
150 | out.println(entryDelimiter);
151 | }
152 | }
153 | }
154 |
155 | void printByActionJson() throws IOException {
156 | JSONArray entries = new JSONArray();
157 | for (String hash : actionMap.keySet()) {
158 | JSONArray actions = new JSONArray();
159 | for (LogEntry entry : actionMap.get(hash).getSortedElements()) {
160 | String s = LogParserUtils.protobufToJsonEntry(entry);
161 | Object obj = JSONValue.parse(s);
162 | actions.add(obj);
163 | }
164 | JSONObject hash_entry = new JSONObject();
165 | hash_entry.put(hash, actions);
166 | entries.add(hash_entry);
167 | }
168 | System.out.println(entries);
169 | }
170 |
171 | List failedActions() throws IOException {
172 | ArrayList result = new ArrayList<>();
173 |
174 | for (String hash : actionMap.keySet()) {
175 | ActionDetails a = actionMap.get(hash);
176 | if (a.isFailed()) {
177 | Digest digest = a.getDigest();
178 | if (digest == null) {
179 | System.err.println("Error: missing digest for failed action " + hash);
180 | } else {
181 | result.add(digest);
182 | }
183 | }
184 | }
185 |
186 | return result;
187 | }
188 |
189 | public static class Builder {
190 | private Map actionMap = new LinkedHashMap<>();
191 |
192 | // The number of entries skipped
193 | private int numSkipped = 0;
194 |
195 | void addLogEntry(LogEntry entry) throws IOException {
196 | if (!entry.hasMetadata()) {
197 | numSkipped++;
198 | return;
199 | }
200 | String hash = entry.getMetadata().getActionId();
201 |
202 | if (!actionMap.containsKey(hash)) {
203 | actionMap.put(hash, new ActionDetails.Builder(hash));
204 | }
205 | actionMap.get(hash).add(entry);
206 | }
207 |
208 | public ActionGrouping build() {
209 | if (numSkipped > 0) {
210 | System.err.printf(
211 | "WARNING: Skipped %d entrie(s) due to absence of request metadata.\n", numSkipped);
212 | }
213 | Map builtActionMap =
214 | actionMap.entrySet().stream()
215 | .collect(
216 | Collectors.toMap(
217 | Map.Entry::getKey,
218 | e -> e.getValue().build(),
219 | (u, v) -> {
220 | throw new IllegalStateException(String.format("Duplicate key %s", u));
221 | },
222 | LinkedHashMap::new));
223 |
224 | return new ActionGrouping(builtActionMap);
225 | }
226 | }
227 | }
228 |
--------------------------------------------------------------------------------
/src/main/java/com/google/devtools/build/remote/client/AuthAndTLSOptions.java:
--------------------------------------------------------------------------------
1 | // Copyright 2018 The Bazel Authors. All rights reserved.
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package com.google.devtools.build.remote.client;
16 |
17 | import com.beust.jcommander.IStringConverter;
18 | import com.beust.jcommander.Parameter;
19 | import com.beust.jcommander.Parameters;
20 | import com.google.common.base.Splitter;
21 | import com.google.common.collect.ImmutableList;
22 | import java.util.List;
23 |
24 | /** Common options for authentication and TLS. */
25 | @Parameters(separators = "=")
26 | public final class AuthAndTLSOptions {
27 | @Parameter(
28 | names = {"--google_default_credentials", "--auth_enabled"},
29 | arity = 1,
30 | description =
31 | "Whether to use 'Google Application Default Credentials' for authentication."
32 | + " See https://cloud.google.com/docs/authentication for details. Disabled by default.")
33 | public boolean useGoogleDefaultCredentials = false;
34 |
35 | @Parameter(
36 | names = "--google_auth_scopes",
37 | listConverter = CommaSeparatedOptionListConverter.class,
38 | description = "A comma-separated list of Google Cloud authentication scopes.")
39 | public List googleAuthScopes =
40 | ImmutableList.of("https://www.googleapis.com/auth/cloud-platform");
41 |
42 | @Parameter(
43 | names = "--google_credentials",
44 | description =
45 | "Specifies the file to get authentication credentials from. See "
46 | + "https://cloud.google.com/docs/authentication for details")
47 | public String googleCredentials = null;
48 |
49 | @Parameter(names = "--tls_enabled", description = "Specifies whether to use TLS.")
50 | public boolean tlsEnabled = false;
51 |
52 | @Parameter(
53 | names = "--tls_certificate",
54 | description = "Specify the TLS client certificate to use.")
55 | public String tlsCertificate = null;
56 |
57 | @Parameter(
58 | names = "--tls_authority_override",
59 | description =
60 | "TESTING ONLY! Can be used with a self-signed certificate to consider the specified "
61 | + "value a valid TLS authority.")
62 | public String tlsAuthorityOverride = null;
63 |
64 | /** A converter for splitting comma-separated string inputs into lists of strings. */
65 | public static class CommaSeparatedOptionListConverter implements IStringConverter> {
66 | @Override
67 | public List convert(String input) {
68 | if (input.isEmpty()) {
69 | return ImmutableList.of();
70 | } else {
71 | return ImmutableList.copyOf(Splitter.on(',').split(input));
72 | }
73 | }
74 | }
75 | }
76 |
--------------------------------------------------------------------------------
/src/main/java/com/google/devtools/build/remote/client/BUILD:
--------------------------------------------------------------------------------
1 | licenses(["notice"]) # Apache 2.0
2 |
3 | java_library(
4 | name = "client",
5 | srcs = glob(["*.java"]),
6 | visibility = ["//:__subpackages__"],
7 | runtime_deps = [
8 | # Needed for tls client.
9 | "@maven//:io_netty_netty_tcnative_boringssl_static",
10 | ],
11 | deps = [
12 | "//src/main/proto:remote_execution_log_java_proto",
13 | "//third_party/remote-apis:build_bazel_remote_execution_v2_remote_execution_java_grpc",
14 | "@com_google_protobuf//:protobuf_java",
15 | "@com_google_protobuf//:protobuf_java_util",
16 | "@googleapis//google/bytestream:bytestream_java_grpc",
17 | "@googleapis//google/bytestream:bytestream_java_proto",
18 | "@googleapis//google/longrunning:longrunning_java_proto",
19 | "@maven//:com_beust_jcommander",
20 | "@maven//:com_google_auth_google_auth_library_credentials",
21 | "@maven//:com_google_auth_google_auth_library_oauth2_http",
22 | "@maven//:com_google_code_findbugs_jsr305",
23 | "@maven//:com_google_guava_guava",
24 | "@maven//:com_googlecode_json_simple_json_simple",
25 | "@maven//:io_grpc_grpc_api",
26 | "@maven//:io_grpc_grpc_auth",
27 | "@maven//:io_grpc_grpc_context",
28 | "@maven//:io_grpc_grpc_core",
29 | # "@maven//:io_grpc_grpc_core_util",
30 | "@maven//:io_grpc_grpc_netty",
31 | "@maven//:io_grpc_grpc_protobuf",
32 | "@maven//:io_grpc_grpc_stub",
33 | "@maven//:io_netty_netty_handler",
34 | "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_java_proto",
35 | "@bazel_remote_apis//build/bazel/semver:semver_java_proto",
36 | ],
37 | )
38 |
--------------------------------------------------------------------------------
/src/main/java/com/google/devtools/build/remote/client/CacheNotFoundException.java:
--------------------------------------------------------------------------------
1 | // Copyright 2018 The Bazel Authors. All rights reserved.
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package com.google.devtools.build.remote.client;
16 |
17 | import build.bazel.remote.execution.v2.Digest;
18 | import java.io.IOException;
19 |
20 | /**
21 | * An exception to indicate cache misses. TODO(olaola): have a class of checked
22 | * RemoteCacheExceptions.
23 | */
24 | public final class CacheNotFoundException extends IOException {
25 | private final Digest missingDigest;
26 |
27 | CacheNotFoundException(Digest missingDigest) {
28 | super("Missing digest: " + missingDigest);
29 | this.missingDigest = missingDigest;
30 | }
31 |
32 | public Digest getMissingDigest() {
33 | return missingDigest;
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/src/main/java/com/google/devtools/build/remote/client/DigestUtil.java:
--------------------------------------------------------------------------------
1 | // Copyright 2018 The Bazel Authors. All rights reserved.
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package com.google.devtools.build.remote.client;
16 |
17 | import static com.google.common.io.MoreFiles.asByteSource;
18 | import static java.nio.charset.StandardCharsets.UTF_8;
19 |
20 | import build.bazel.remote.execution.v2.Digest;
21 | import com.google.common.hash.HashCode;
22 | import com.google.common.hash.HashFunction;
23 | import com.google.protobuf.Message;
24 | import java.io.IOException;
25 | import java.nio.file.Files;
26 | import java.nio.file.Path;
27 |
28 | public class DigestUtil {
29 | private final HashFunction hashFn;
30 |
31 | public DigestUtil(HashFunction hashFn) {
32 | this.hashFn = hashFn;
33 | }
34 |
35 | public Digest compute(byte[] blob) {
36 | return buildDigest(hashFn.hashBytes(blob).toString(), blob.length);
37 | }
38 |
39 | /**
40 | * Computes a digest of the given proto message. Currently, we simply rely on message output as
41 | * bytes, but this implementation relies on the stability of the proto encoding, in particular
42 | * between different platforms and languages.
43 | */
44 | public Digest compute(Message message) {
45 | return compute(message.toByteArray());
46 | }
47 |
48 | public Digest computeAsUtf8(String str) {
49 | return compute(str.getBytes(UTF_8));
50 | }
51 |
52 | public Digest compute(Path path) throws IOException {
53 | if (!Files.isRegularFile(path)) {
54 | throw new IOException("Only can compute hash for regular file.");
55 | }
56 | long fileSize = Files.size(path);
57 | return buildDigest(asByteSource(path).hash(hashFn).asBytes(), fileSize);
58 | }
59 |
60 | public static Digest buildDigest(byte[] hash, long size) {
61 | return buildDigest(HashCode.fromBytes(hash).toString(), size);
62 | }
63 |
64 | public static Digest buildDigest(String hexHash, long size) {
65 | return Digest.newBuilder().setHash(hexHash).setSizeBytes(size).build();
66 | }
67 |
68 | public String toString(Digest digest) {
69 | return digest.getHash() + "/" + digest.getSizeBytes();
70 | }
71 | }
72 |
--------------------------------------------------------------------------------
/src/main/java/com/google/devtools/build/remote/client/DockerUtil.java:
--------------------------------------------------------------------------------
1 | // Copyright 2018 The Bazel Authors. All rights reserved.
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | package com.google.devtools.build.remote.client;
15 |
16 | import build.bazel.remote.execution.v2.Command;
17 | import build.bazel.remote.execution.v2.Command.EnvironmentVariable;
18 | import build.bazel.remote.execution.v2.Platform;
19 | import com.google.common.annotations.VisibleForTesting;
20 | import com.google.common.io.ByteStreams;
21 | import java.io.IOException;
22 | import java.io.InputStream;
23 | import java.util.ArrayList;
24 | import java.util.List;
25 | import javax.annotation.Nullable;
26 |
27 | public final class DockerUtil {
28 | private static final String CONTAINER_IMAGE_ENTRY_NAME = "container-image";
29 | private static final String DOCKER_IMAGE_PREFIX = "docker://";
30 |
31 | @VisibleForTesting
32 | static class UidGetter {
33 | /**
34 | * Gets uid of the current user. If the uid could not be fetched, prints a message to stderr and
35 | * returns -1.
36 | */
37 | @VisibleForTesting
38 | long getUid() {
39 | ProcessBuilder processBuilder = new ProcessBuilder();
40 | processBuilder.command("id", "-u");
41 | try {
42 | InputStream stdout = processBuilder.start().getInputStream();
43 | byte[] output = ByteStreams.toByteArray(stdout);
44 | return Long.parseLong(new String(output).trim());
45 | } catch (IOException | NumberFormatException e) {
46 | System.err.printf(
47 | "Could not fetch UID for passing to Docker container. The provided docker "
48 | + "command will not specify a uid (error: %s)\n",
49 | e.toString());
50 | return -1;
51 | }
52 | }
53 | }
54 |
55 | private UidGetter uidGetter;
56 |
57 | @VisibleForTesting
58 | DockerUtil(UidGetter getter) {
59 | uidGetter = getter;
60 | }
61 |
62 | DockerUtil() {
63 | this(new UidGetter());
64 | }
65 |
66 | /**
67 | * Checks Action for Docker container definition.
68 | *
69 | * @return The docker container for the command. If no container could be found, returns null.
70 | */
71 | private static @Nullable String dockerContainer(Command command) {
72 | String result = null;
73 | for (Platform.Property property : command.getPlatform().getPropertiesList()) {
74 | if (property.getName().equals(CONTAINER_IMAGE_ENTRY_NAME)) {
75 | if (result != null) {
76 | // Multiple container name entries
77 | throw new IllegalArgumentException(
78 | String.format(
79 | "Multiple entries for %s in command.Platform", CONTAINER_IMAGE_ENTRY_NAME));
80 | }
81 | result = property.getValue();
82 | if (!result.startsWith(DOCKER_IMAGE_PREFIX)) {
83 | throw new IllegalArgumentException(
84 | String.format(
85 | "%s: Docker images must be stored in gcr.io with an image spec in the form "
86 | + "'docker://gcr.io/{IMAGE_NAME}'",
87 | CONTAINER_IMAGE_ENTRY_NAME));
88 | }
89 | result = result.substring(DOCKER_IMAGE_PREFIX.length());
90 | }
91 | }
92 | return result;
93 | }
94 |
95 | /**
96 | * Outputs a Docker command that will execute the given action in the given path.
97 | *
98 | * @param action The Action to be executed in the output docker container command.
99 | * @param command The Command of the Action being executed. This must match the Command that is
100 | * referred to from the input parameter Action.
101 | * @param workingPath The path that is to be the working directory that the Action is to be
102 | * executed in.
103 | */
104 | public String getDockerCommand(Command command, String workingPath) {
105 | String container = dockerContainer(command);
106 | if (container == null) {
107 | throw new IllegalArgumentException("No docker image specified in given Command.");
108 | }
109 | List commandElements = new ArrayList<>();
110 | commandElements.add("docker");
111 | commandElements.add("run");
112 |
113 | long uid = uidGetter.getUid();
114 | if (uid >= 0 && !System.getProperty("os.name").startsWith("Windows")) {
115 | commandElements.add("-u");
116 | commandElements.add(Long.toString(uid));
117 | }
118 |
119 | String dockerPathString = workingPath + "-docker";
120 | commandElements.add("-v");
121 | commandElements.add(workingPath + ":" + dockerPathString);
122 | commandElements.add("-w");
123 | commandElements.add(dockerPathString);
124 |
125 | for (EnvironmentVariable var : command.getEnvironmentVariablesList()) {
126 | commandElements.add("-e");
127 | commandElements.add(var.getName() + "=" + var.getValue());
128 | }
129 |
130 | commandElements.add(container);
131 | commandElements.addAll(command.getArgumentsList());
132 |
133 | return ShellEscaper.escapeJoinAll(commandElements);
134 | }
135 | }
136 |
--------------------------------------------------------------------------------
/src/main/java/com/google/devtools/build/remote/client/GoogleAuthUtils.java:
--------------------------------------------------------------------------------
1 | // Copyright 2018 The Bazel Authors. All rights reserved.
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package com.google.devtools.build.remote.client;
16 |
17 | import com.google.auth.Credentials;
18 | import com.google.auth.oauth2.GoogleCredentials;
19 | import com.google.common.annotations.VisibleForTesting;
20 | import com.google.common.base.Preconditions;
21 | import io.grpc.CallCredentials;
22 | import io.grpc.ManagedChannel;
23 | import io.grpc.auth.MoreCallCredentials;
24 | import io.grpc.netty.GrpcSslContexts;
25 | import io.grpc.netty.NegotiationType;
26 | import io.grpc.netty.NettyChannelBuilder;
27 | import io.netty.handler.ssl.SslContext;
28 | import java.io.File;
29 | import java.io.FileInputStream;
30 | import java.io.FileNotFoundException;
31 | import java.io.IOException;
32 | import java.io.InputStream;
33 | import java.util.List;
34 | import javax.annotation.Nullable;
35 |
36 | /**
37 | * Utility methods for using {@link AuthAndTLSOptions} with Google Cloud.
38 | *
39 | * This code in this file was directly copied from the main Bazel git repository at commit hash
40 | * e24fe4dbc62e2c9081e915b5318ed3e5ee47a76a.
41 | */
42 | public final class GoogleAuthUtils {
43 |
44 | /**
45 | * Create a new gRPC {@link ManagedChannel}.
46 | *
47 | * @throws IOException in case the channel can't be constructed.
48 | */
49 | public static ManagedChannel newChannel(String target, AuthAndTLSOptions options)
50 | throws IOException {
51 | Preconditions.checkNotNull(target);
52 | Preconditions.checkNotNull(options);
53 |
54 | final SslContext sslContext =
55 | options.tlsEnabled ? createSSlContext(options.tlsCertificate) : null;
56 |
57 | try {
58 | NettyChannelBuilder builder =
59 | NettyChannelBuilder.forTarget(target)
60 | .negotiationType(options.tlsEnabled ? NegotiationType.TLS : NegotiationType.PLAINTEXT)
61 | .defaultLoadBalancingPolicy("round_robin");
62 | if (sslContext != null) {
63 | builder.sslContext(sslContext);
64 | if (options.tlsAuthorityOverride != null) {
65 | builder.overrideAuthority(options.tlsAuthorityOverride);
66 | }
67 | }
68 | return builder.build();
69 | } catch (RuntimeException e) {
70 | // gRPC might throw all kinds of RuntimeExceptions: StatusRuntimeException,
71 | // IllegalStateException, NullPointerException, ...
72 | String message = "Failed to connect to '%s': %s";
73 | throw new IOException(String.format(message, target, e.getMessage()));
74 | }
75 | }
76 |
77 | private static SslContext createSSlContext(@Nullable String rootCert) throws IOException {
78 | if (rootCert == null) {
79 | try {
80 | return GrpcSslContexts.forClient().build();
81 | } catch (Exception e) {
82 | String message = "Failed to init TLS infrastructure: " + e.getMessage();
83 | throw new IOException(message, e);
84 | }
85 | } else {
86 | try {
87 | return GrpcSslContexts.forClient().trustManager(new File(rootCert)).build();
88 | } catch (Exception e) {
89 | String message = "Failed to init TLS infrastructure using '%s' as root certificate: %s";
90 | message = String.format(message, rootCert, e.getMessage());
91 | throw new IOException(message, e);
92 | }
93 | }
94 | }
95 |
96 | /**
97 | * Create a new {@link CallCredentials} object.
98 | *
99 | * @throws IOException in case the call credentials can't be constructed.
100 | */
101 | public static CallCredentials newCallCredentials(AuthAndTLSOptions options) throws IOException {
102 | Credentials creds = newCredentials(options);
103 | if (creds != null) {
104 | return MoreCallCredentials.from(creds);
105 | }
106 | return null;
107 | }
108 |
109 | @VisibleForTesting
110 | public static CallCredentials newCallCredentials(
111 | @Nullable InputStream credentialsFile, List authScope) throws IOException {
112 | Credentials creds = newCredentials(credentialsFile, authScope);
113 | if (creds != null) {
114 | return MoreCallCredentials.from(creds);
115 | }
116 | return null;
117 | }
118 |
119 | /**
120 | * Create a new {@link Credentials} object.
121 | *
122 | * @throws IOException in case the credentials can't be constructed.
123 | */
124 | public static Credentials newCredentials(AuthAndTLSOptions options) throws IOException {
125 | if (options.googleCredentials != null) {
126 | // Credentials from file
127 | try (InputStream authFile = new FileInputStream(options.googleCredentials)) {
128 | return newCredentials(authFile, options.googleAuthScopes);
129 | } catch (FileNotFoundException e) {
130 | String message =
131 | String.format(
132 | "Could not open auth credentials file '%s': %s",
133 | options.googleCredentials, e.getMessage());
134 | throw new IOException(message, e);
135 | }
136 | } else if (options.useGoogleDefaultCredentials) {
137 | return newCredentials(
138 | null /* Google Application Default Credentials */, options.googleAuthScopes);
139 | }
140 |
141 | return null;
142 | }
143 |
144 | private static Credentials newCredentials(
145 | @Nullable InputStream credentialsFile, List authScopes) throws IOException {
146 | try {
147 | GoogleCredentials creds =
148 | credentialsFile == null
149 | ? GoogleCredentials.getApplicationDefault()
150 | : GoogleCredentials.fromStream(credentialsFile);
151 | if (!authScopes.isEmpty()) {
152 | creds = creds.createScoped(authScopes);
153 | }
154 | return creds;
155 | } catch (IOException e) {
156 | String message = "Failed to init auth credentials: " + e.getMessage();
157 | throw new IOException(message, e);
158 | }
159 | }
160 | }
161 |
--------------------------------------------------------------------------------
/src/main/java/com/google/devtools/build/remote/client/GrpcRemoteCache.java:
--------------------------------------------------------------------------------
1 | // Copyright 2018 The Bazel Authors. All rights reserved.
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package com.google.devtools.build.remote.client;
16 |
17 | import build.bazel.remote.execution.v2.ContentAddressableStorageGrpc;
18 | import build.bazel.remote.execution.v2.ContentAddressableStorageGrpc.ContentAddressableStorageBlockingStub;
19 | import build.bazel.remote.execution.v2.Digest;
20 | import build.bazel.remote.execution.v2.Directory;
21 | import build.bazel.remote.execution.v2.GetTreeRequest;
22 | import build.bazel.remote.execution.v2.GetTreeResponse;
23 | import build.bazel.remote.execution.v2.Tree;
24 | import com.google.bytestream.ByteStreamGrpc;
25 | import com.google.bytestream.ByteStreamGrpc.ByteStreamBlockingStub;
26 | import com.google.bytestream.ByteStreamProto.ReadRequest;
27 | import com.google.bytestream.ByteStreamProto.ReadResponse;
28 | import io.grpc.CallCredentials;
29 | import io.grpc.ClientInterceptor;
30 | import io.grpc.Channel;
31 | import io.grpc.Metadata;
32 | import io.grpc.Status;
33 | import io.grpc.StatusRuntimeException;
34 | import io.grpc.stub.MetadataUtils;
35 | import java.io.ByteArrayOutputStream;
36 | import java.io.IOException;
37 | import java.io.OutputStream;
38 | import java.nio.file.Files;
39 | import java.nio.file.Path;
40 | import java.util.Iterator;
41 | import java.util.Map;
42 | import java.util.concurrent.TimeUnit;
43 |
44 | /** A RemoteActionCache implementation that uses gRPC calls to a remote cache server. */
45 | public class GrpcRemoteCache extends AbstractRemoteActionCache {
46 |
47 | private final RemoteOptions options;
48 | private final CallCredentials credentials;
49 | private final Channel channel;
50 |
51 | public GrpcRemoteCache(
52 | RemoteOptions options, AuthAndTLSOptions authAndTLSOptions, DigestUtil digestUtil)
53 | throws IOException {
54 | this(
55 | GoogleAuthUtils.newChannel(options.remoteCache, authAndTLSOptions),
56 | GoogleAuthUtils.newCallCredentials(authAndTLSOptions),
57 | options,
58 | digestUtil);
59 | }
60 |
61 | public GrpcRemoteCache(
62 | Channel channel, CallCredentials credentials, RemoteOptions options, DigestUtil digestUtil) {
63 | super(digestUtil);
64 | this.options = options;
65 | this.credentials = credentials;
66 | this.channel = channel;
67 | }
68 |
69 | public static boolean isRemoteCacheOptions(RemoteOptions options) {
70 | return options.remoteCache != null;
71 | }
72 |
73 | private static ClientInterceptor customHeadersInterceptor(Map headers) {
74 | Metadata metadata = new Metadata();
75 | for (Map.Entry entry : headers.entrySet()) {
76 | metadata.put(
77 | Metadata.Key.of(entry.getKey(), Metadata.ASCII_STRING_MARSHALLER),
78 | entry.getValue()
79 | );
80 | }
81 | return MetadataUtils.newAttachHeadersInterceptor(metadata);
82 | }
83 |
84 | private ContentAddressableStorageBlockingStub casBlockingStub() {
85 | return ContentAddressableStorageGrpc.newBlockingStub(channel)
86 | .withInterceptors(
87 | TracingMetadataUtils.attachMetadataFromContextInterceptor(),
88 | customHeadersInterceptor(options.remoteHeaders)
89 | )
90 | .withCallCredentials(credentials)
91 | .withDeadlineAfter(options.remoteTimeout, TimeUnit.SECONDS);
92 | }
93 |
94 | private ByteStreamBlockingStub bsBlockingStub() {
95 | return ByteStreamGrpc.newBlockingStub(channel)
96 | .withInterceptors(
97 | TracingMetadataUtils.attachMetadataFromContextInterceptor(),
98 | customHeadersInterceptor(options.remoteHeaders)
99 | )
100 | .withCallCredentials(credentials)
101 | .withDeadlineAfter(options.remoteTimeout, TimeUnit.SECONDS);
102 | }
103 |
104 | /**
105 | * Download a tree with the {@link Directory} given by digest as the root directory of the tree.
106 | * This method attempts to retrieve the {@link Tree} using the GetTree RPC.
107 | *
108 | * @param rootDigest The digest of the root {@link Directory} of the tree
109 | * @return A tree with the given directory as the root.
110 | * @throws IOException in the case that retrieving the blobs required to reconstruct the tree
111 | * failed.
112 | */
113 | @Override
114 | public Tree getTree(Digest rootDigest) throws IOException {
115 | Directory dir;
116 | try {
117 | dir = Directory.parseFrom(downloadBlob(rootDigest));
118 | } catch (IOException e) {
119 | throw new IOException("Failed to download root Directory of tree.", e);
120 | }
121 |
122 | Tree.Builder result = Tree.newBuilder().setRoot(dir);
123 |
124 | GetTreeRequest.Builder requestBuilder =
125 | GetTreeRequest.newBuilder()
126 | .setRootDigest(rootDigest)
127 | .setInstanceName(options.remoteInstanceName);
128 |
129 | Iterator responses = casBlockingStub().getTree(requestBuilder.build());
130 | while (responses.hasNext()) {
131 | result.addAllChildren(responses.next().getDirectoriesList());
132 | }
133 |
134 | return result.build();
135 | }
136 |
137 | @Override
138 | protected void downloadBlob(Digest digest, Path dest) throws IOException {
139 | try (OutputStream out = Files.newOutputStream(dest)) {
140 | readBlob(digest, out);
141 | }
142 | }
143 |
144 | @Override
145 | protected byte[] downloadBlob(Digest digest) throws IOException {
146 | if (digest.getSizeBytes() == 0) {
147 | return new byte[0];
148 | }
149 | ByteArrayOutputStream stream = new ByteArrayOutputStream((int) digest.getSizeBytes());
150 | readBlob(digest, stream);
151 | return stream.toByteArray();
152 | }
153 |
154 | @Override
155 | public void downloadBlob(Digest digest, OutputStream stream) throws IOException {
156 | if (digest.getSizeBytes() == 0) {
157 | return;
158 | }
159 | readBlob(digest, stream);
160 | }
161 |
162 | private void readBlob(Digest digest, OutputStream stream) throws IOException {
163 | String resourceName = "";
164 | if (!options.remoteInstanceName.isEmpty()) {
165 | resourceName += options.remoteInstanceName + "/";
166 | }
167 | resourceName += "blobs/" + digest.getHash() + "/" + digest.getSizeBytes();
168 | try {
169 | Iterator replies =
170 | bsBlockingStub().read(ReadRequest.newBuilder().setResourceName(resourceName).build());
171 | while (replies.hasNext()) {
172 | replies.next().getData().writeTo(stream);
173 | }
174 | } catch (StatusRuntimeException e) {
175 | if (e.getStatus().getCode() == Status.Code.NOT_FOUND) {
176 | throw new CacheNotFoundException(digest);
177 | }
178 | throw e;
179 | }
180 | }
181 | }
182 |
--------------------------------------------------------------------------------
/src/main/java/com/google/devtools/build/remote/client/LogParserUtils.java:
--------------------------------------------------------------------------------
1 | // Copyright 2018 The Bazel Authors. All rights reserved.
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package com.google.devtools.build.remote.client;
16 |
17 | import static java.nio.charset.StandardCharsets.UTF_8;
18 | import static com.google.common.base.Preconditions.checkNotNull;
19 |
20 | import build.bazel.remote.execution.v2.ActionResult;
21 | import build.bazel.remote.execution.v2.Digest;
22 | import build.bazel.remote.execution.v2.ExecuteResponse;
23 | import build.bazel.remote.execution.v2.ExecuteOperationMetadata;
24 | import build.bazel.remote.execution.v2.ExecutedActionMetadata;
25 | import com.google.devtools.build.lib.remote.logging.RemoteExecutionLog.LogEntry;
26 | import com.google.devtools.build.lib.remote.logging.RemoteExecutionLog.RpcCallDetails;
27 | import com.google.devtools.build.remote.client.RemoteClientOptions.PrintLogCommand;
28 | import com.google.longrunning.Operation;
29 | import com.google.longrunning.Operation.ResultCase;
30 | import com.google.protobuf.Any;
31 | import com.google.protobuf.Descriptors.Descriptor;
32 | import com.google.protobuf.InvalidProtocolBufferException;
33 | import com.google.protobuf.Message;
34 | import com.google.protobuf.StringValue;
35 | import com.google.protobuf.util.JsonFormat;
36 | import io.grpc.Status.Code;
37 | import java.io.BufferedWriter;
38 | import java.io.FileInputStream;
39 | import java.io.IOException;
40 | import java.io.InputStream;
41 | import java.io.OutputStream;
42 | import java.io.OutputStreamWriter;
43 | import java.io.PrintWriter;
44 | import java.util.ArrayList;
45 | import java.util.Arrays;
46 | import java.util.Collections;
47 | import java.util.List;
48 | import org.json.simple.JSONArray;
49 | import org.json.simple.JSONObject;
50 | import org.json.simple.JSONValue;
51 |
52 | /** Methods for printing log files. */
53 | public class LogParserUtils {
54 |
55 | private static final String DELIMETER =
56 | "---------------------------------------------------------\n";
57 |
58 | public static class ParamException extends Exception {
59 |
60 | public ParamException(String message) {
61 | super(message);
62 | }
63 | }
64 |
65 | private String filename;
66 |
67 | public LogParserUtils(String filename) {
68 | this.filename = filename;
69 | }
70 |
71 | private FileInputStream openGrpcFileInputStream() throws ParamException, IOException {
72 | if (filename.isEmpty()) {
73 | throw new ParamException("This operation cannot be performed without specifying --grpc_log.");
74 | }
75 | return new FileInputStream(filename);
76 | }
77 |
78 | // Returns an ExecuteResponse contained in the operation and null if none.
79 | // If the operation contains an error, returns null and populates StringBuilder "error" with
80 | // the error message.
81 | public static T getExecuteResponse(
82 | Operation o, Class t, StringBuilder error) throws IOException {
83 | if (o.getResultCase() == ResultCase.ERROR && o.getError().getCode() != Code.OK.value()) {
84 | error.append(o.getError().toString());
85 | return null;
86 | }
87 | if (o.getResultCase() == ResultCase.RESPONSE && o.getDone()) {
88 | return o.getResponse().unpack(t);
89 | }
90 | return null;
91 | }
92 |
93 | // Returns an ExecuteOperationMetadata contained in the operation and null if none.
94 | // If the operation contains an error, returns null and populates StringBuilder "error" with
95 | // the error message.
96 | public static T getExecuteMetadata(
97 | Operation o, Class t, StringBuilder error) throws IOException {
98 | try {
99 | return o.getMetadata().unpack(t);
100 | } catch (InvalidProtocolBufferException e) {
101 | return null;
102 | }
103 | }
104 |
105 | // Returns a Digest contained in the log entry, and null if none.
106 | public static Digest extractDigest(LogEntry entry) {
107 | if (!entry.hasDetails()) {
108 | return null;
109 | }
110 | RpcCallDetails details = entry.getDetails();
111 | if (details.hasExecute()) {
112 | if (details.getExecute().hasRequest()
113 | && details.getExecute().getRequest().hasActionDigest()) {
114 | return details.getExecute().getRequest().getActionDigest();
115 | }
116 | }
117 | if (details.hasGetActionResult()) {
118 | if (details.getGetActionResult().hasRequest()
119 | && details.getGetActionResult().getRequest().hasActionDigest()) {
120 | return details.getGetActionResult().getRequest().getActionDigest();
121 | }
122 | }
123 | return null;
124 | }
125 |
126 | private static List extractExecuteResponse(List operations)
127 | throws IOException {
128 | ArrayList result = new ArrayList<>();
129 | for (Operation o : operations) {
130 | StringBuilder error = new StringBuilder();
131 | ExecuteResponse response = getExecuteResponse(o, ExecuteResponse.class, error);
132 | if (response != null
133 | && (response.hasResult()
134 | || (response.hasStatus()) && response.getStatus().getCode() != Code.OK.value())) {
135 | result.add(response);
136 | }
137 | }
138 | return result;
139 | }
140 |
141 | // Returns a list of ExecuteResponse messages contained in the log entry,
142 | // an empty list if there are none.
143 | // If the LogEntry contains a successful cache lookup, an ExecuteResponse is constructed
144 | // with that ActionResult and cached_result set to true.
145 | public static List extractExecuteResponse(LogEntry entry) throws IOException {
146 | if (!entry.hasDetails()) {
147 | return Collections.emptyList();
148 | }
149 | if (entry.getStatus().getCode() != Code.OK.value()) {
150 | return Collections.emptyList();
151 | }
152 | RpcCallDetails details = entry.getDetails();
153 | if (details.hasExecute()) {
154 | return extractExecuteResponse(details.getExecute().getResponsesList());
155 | } else if (details.hasWaitExecution()) {
156 | return extractExecuteResponse(details.getWaitExecution().getResponsesList());
157 | } else if (details.hasGetActionResult()) {
158 | ExecuteResponse response =
159 | ExecuteResponse.newBuilder()
160 | .setResult(details.getGetActionResult().getResponse())
161 | .setCachedResult(true)
162 | .build();
163 | return Arrays.asList(response);
164 | }
165 | return Collections.emptyList();
166 | }
167 |
168 | private static boolean maybePrintOperation(
169 | Operation o, PrintWriter out, Class t) throws IOException {
170 | StringBuilder error = new StringBuilder();
171 | T result = getExecuteResponse(o, t, error);
172 | if (result != null) {
173 | out.println("ExecuteResponse extracted:");
174 | out.println(result.toString());
175 | return true;
176 | }
177 | String errString = error.toString();
178 | if (!errString.isEmpty()) {
179 | out.printf("Operation contained error: %s\n", o.getError().toString());
180 | return true;
181 | }
182 | return false;
183 | }
184 |
185 | private static boolean maybePrintMetadata(
186 | Operation o, PrintWriter out, Class t) throws IOException {
187 | StringBuilder error = new StringBuilder();
188 | T result = getExecuteMetadata(o, t, error);
189 | if (result != null) {
190 | out.println("Metadata extracted:");
191 | out.println(result.toString());
192 | return true;
193 | }
194 | return false;
195 | }
196 |
197 | /** Print execute responses or errors contained in the given list of operations. */
198 | private static void printExecuteResponse(List operations, PrintWriter out)
199 | throws IOException {
200 | for (Operation o : operations) {
201 | maybePrintOperation(o, out, build.bazel.remote.execution.v2.ExecuteResponse.class);
202 | maybePrintMetadata(o, out, build.bazel.remote.execution.v2.ExecuteOperationMetadata.class);
203 | }
204 | }
205 | /** Print an individual log entry. */
206 | static void printLogEntry(LogEntry entry, PrintWriter out) throws IOException {
207 | out.println(entry.toString());
208 |
209 | switch (entry.getDetails().getDetailsCase()) {
210 | case EXECUTE:
211 | out.println(
212 | "\nAttempted to extract ExecuteResponse from streaming Execute call responses:");
213 | printExecuteResponse(entry.getDetails().getExecute().getResponsesList(), out);
214 | break;
215 | case WAIT_EXECUTION:
216 | out.println(
217 | "\nAttempted to extract ExecuteResponse from streaming WaitExecution call responses:");
218 | printExecuteResponse(entry.getDetails().getWaitExecution().getResponsesList(), out);
219 | break;
220 | }
221 | }
222 |
223 | static String protobufToJsonEntry(LogEntry input) throws InvalidProtocolBufferException {
224 | return JsonFormat.printer()
225 | .usingTypeRegistry(
226 | JsonFormat.TypeRegistry.newBuilder()
227 | .add(ExecuteOperationMetadata.getDescriptor())
228 | .build())
229 | .print(checkNotNull(input));
230 | }
231 |
232 | /**
233 | * Prints each entry out individually (ungrouped) and a message at the end for how many entries
234 | * were printed/skipped.
235 | */
236 | private void printEntriesInOrder(OutputStream outStream) throws IOException, ParamException {
237 | try (InputStream in = openGrpcFileInputStream()) {
238 | PrintWriter out =
239 | new PrintWriter(new BufferedWriter(new OutputStreamWriter(outStream, UTF_8)), true);
240 | LogEntry entry;
241 | while ((entry = LogEntry.parseDelimitedFrom(in)) != null) {
242 | printLogEntry(entry, out);
243 | System.out.print(DELIMETER);
244 | }
245 | }
246 | }
247 |
248 | private List getResponses(RpcCallDetails details) {
249 | switch (details.getDetailsCase()) {
250 | case EXECUTE:
251 | return details.getExecute().getResponsesList();
252 | case WAIT_EXECUTION:
253 | return details.getWaitExecution().getResponsesList();
254 | }
255 | return null;
256 | }
257 |
258 | private void filterExecutedActionMetadata(ExecutedActionMetadata.Builder builder) {
259 | List auxiliaryMetadata = new ArrayList<>(builder.getAuxiliaryMetadataList());
260 | builder.clearAuxiliaryMetadata();
261 | for (Any metadata : auxiliaryMetadata) {
262 | builder.addAuxiliaryMetadata(Any.pack(StringValue.newBuilder().setValue(metadata.toString()).build()));
263 | }
264 | }
265 |
266 | private LogEntry filterUnknownAny(LogEntry entry) throws IOException {
267 | ActionResult result;
268 | LogEntry.Builder builder = entry.toBuilder();
269 | switch (entry.getDetails().getDetailsCase()) {
270 | case EXECUTE:
271 | case WAIT_EXECUTION:
272 | List responses = getResponses(entry.getDetails());
273 | List filteredResponses = new ArrayList<>();
274 | for (Operation response : responses) {
275 | StringBuilder error = new StringBuilder();
276 | ExecuteOperationMetadata metadata = getExecuteMetadata(response, ExecuteOperationMetadata.class, error);
277 | Operation.Builder filteredResponse = response.toBuilder();
278 | if (metadata != null && metadata.hasPartialExecutionMetadata()) {
279 | ExecuteOperationMetadata.Builder metadataBuilder = metadata.toBuilder();
280 | filterExecutedActionMetadata(metadataBuilder.getPartialExecutionMetadataBuilder());
281 | filteredResponse.setMetadata(Any.pack(metadataBuilder.build()));
282 | }
283 | ExecuteResponse executeResponse = getExecuteResponse(response, ExecuteResponse.class, error);
284 | if (executeResponse != null && executeResponse.getResult().hasExecutionMetadata()) {
285 | ExecuteResponse.Builder responseBuilder = executeResponse.toBuilder();
286 | filterExecutedActionMetadata(responseBuilder.getResultBuilder().getExecutionMetadataBuilder());
287 | filteredResponse.setResponse(Any.pack(responseBuilder.build()));
288 | }
289 | filteredResponses.add(filteredResponse.build());
290 | }
291 | switch (entry.getDetails().getDetailsCase()) {
292 | case EXECUTE:
293 | builder.getDetailsBuilder().getExecuteBuilder().clearResponses().addAllResponses(filteredResponses);
294 | break;
295 | case WAIT_EXECUTION:
296 | builder.getDetailsBuilder().getWaitExecutionBuilder().clearResponses().addAllResponses(filteredResponses);
297 | break;
298 | }
299 | return builder.build();
300 | case GET_ACTION_RESULT:
301 | ActionResult.Builder resultBuilder = builder.getDetailsBuilder().getGetActionResultBuilder().getResponseBuilder();
302 | filterExecutedActionMetadata(resultBuilder.getExecutionMetadataBuilder());
303 | return builder.build();
304 | }
305 | return entry;
306 | }
307 |
308 | /**
309 | * Prints each entry out individually (ungrouped) and a message at the end for how many entries
310 | * were printed/skipped.
311 | */
312 | private void printEntriesInJson() throws IOException, ParamException {
313 | try (InputStream in = openGrpcFileInputStream()) {
314 | LogEntry entry;
315 | JSONArray entries = new JSONArray();
316 | while ((entry = LogEntry.parseDelimitedFrom(in)) != null) {
317 | String s = protobufToJsonEntry(filterUnknownAny(entry));
318 | Object obj = JSONValue.parse(s);
319 | entries.add(obj);
320 | }
321 | System.out.print(entries);
322 | }
323 | }
324 |
325 | private ActionGrouping initActionGrouping() throws IOException, ParamException {
326 | ActionGrouping.Builder result = new ActionGrouping.Builder();
327 | try (InputStream in = openGrpcFileInputStream()) {
328 | LogEntry entry;
329 | while ((entry = LogEntry.parseDelimitedFrom(in)) != null) {
330 | result.addLogEntry(entry);
331 | }
332 | }
333 | return result.build();
334 | }
335 |
336 | private void printEntriesGroupedByAction(OutputStream outStream)
337 | throws IOException, ParamException {
338 | ActionGrouping byAction = initActionGrouping();
339 | PrintWriter out =
340 | new PrintWriter(new BufferedWriter(new OutputStreamWriter(outStream, UTF_8)), true);
341 | byAction.printByAction(out);
342 | }
343 |
344 | private void printEntriesGroupedByActionJson()
345 | throws IOException, ParamException {
346 | ActionGrouping byAction = initActionGrouping();
347 | byAction.printByActionJson();
348 | }
349 |
350 | /** Print log entries to standard output according to the command line arguments given. */
351 | public void printLog(PrintLogCommand options) throws IOException {
352 | try {
353 | if (options.formatJson && options.groupByAction){
354 | printEntriesGroupedByActionJson();
355 | } else if (options.formatJson) {
356 | printEntriesInJson();
357 | } else if (options.groupByAction){
358 | printEntriesGroupedByAction(System.out);
359 | } else {
360 | printEntriesInOrder(System.out);
361 | }
362 | } catch (ParamException e) {
363 | System.err.println(e.getMessage());
364 | System.exit(1);
365 | }
366 | }
367 |
368 | /** Returns a list of actions failed in the grpc log */
369 | public List failedActions() throws IOException, ParamException {
370 | ActionGrouping a = initActionGrouping();
371 | return a.failedActions();
372 | }
373 |
374 | /** Print a list of actions */
375 | public void printFailedActions() throws IOException, ParamException {
376 | PrintWriter out =
377 | new PrintWriter(new BufferedWriter(new OutputStreamWriter(System.out, UTF_8)), true);
378 | List actions = failedActions();
379 | if (actions.size() == 0) {
380 | out.println("No failed actions found.");
381 | return;
382 | }
383 | for (Digest d : actions) {
384 | out.println("Failed action: " + d.getHash() + "/" + d.getSizeBytes());
385 | }
386 | }
387 | }
388 |
--------------------------------------------------------------------------------
/src/main/java/com/google/devtools/build/remote/client/RemoteClient.java:
--------------------------------------------------------------------------------
1 | // Copyright 2018 The Bazel Authors. All rights reserved.
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package com.google.devtools.build.remote.client;
16 |
17 | import static java.nio.charset.StandardCharsets.UTF_8;
18 |
19 | import build.bazel.remote.execution.v2.Action;
20 | import build.bazel.remote.execution.v2.ActionResult;
21 | import build.bazel.remote.execution.v2.Command;
22 | import build.bazel.remote.execution.v2.Command.EnvironmentVariable;
23 | import build.bazel.remote.execution.v2.Digest;
24 | import build.bazel.remote.execution.v2.Directory;
25 | import build.bazel.remote.execution.v2.DirectoryNode;
26 | import build.bazel.remote.execution.v2.FileNode;
27 | import build.bazel.remote.execution.v2.OutputDirectory;
28 | import build.bazel.remote.execution.v2.OutputFile;
29 | import build.bazel.remote.execution.v2.Platform;
30 | import build.bazel.remote.execution.v2.RequestMetadata;
31 | import build.bazel.remote.execution.v2.ToolDetails;
32 | import build.bazel.remote.execution.v2.Tree;
33 | import com.beust.jcommander.JCommander;
34 | import com.beust.jcommander.ParameterException;
35 | import com.google.common.hash.Hashing;
36 | import com.google.common.io.Files;
37 | import com.google.devtools.build.remote.client.LogParserUtils.ParamException;
38 | import com.google.devtools.build.remote.client.RemoteClientOptions.CatCommand;
39 | import com.google.devtools.build.remote.client.RemoteClientOptions.FailedActionsCommand;
40 | import com.google.devtools.build.remote.client.RemoteClientOptions.GetDirCommand;
41 | import com.google.devtools.build.remote.client.RemoteClientOptions.GetOutDirCommand;
42 | import com.google.devtools.build.remote.client.RemoteClientOptions.LsCommand;
43 | import com.google.devtools.build.remote.client.RemoteClientOptions.LsOutDirCommand;
44 | import com.google.devtools.build.remote.client.RemoteClientOptions.PrintLogCommand;
45 | import com.google.devtools.build.remote.client.RemoteClientOptions.RunCommand;
46 | import com.google.devtools.build.remote.client.RemoteClientOptions.ShowActionCommand;
47 | import com.google.devtools.build.remote.client.RemoteClientOptions.ShowActionResultCommand;
48 | import com.google.protobuf.TextFormat;
49 | import io.grpc.Context;
50 | import io.grpc.Status;
51 | import java.io.File;
52 | import java.io.FileInputStream;
53 | import java.io.FileOutputStream;
54 | import java.io.IOException;
55 | import java.io.InputStreamReader;
56 | import java.io.OutputStream;
57 | import java.nio.file.FileSystemAlreadyExistsException;
58 | import java.nio.file.Path;
59 | import java.nio.file.Paths;
60 | import java.util.HashMap;
61 | import java.util.List;
62 | import java.util.Map;
63 |
64 | /** A standalone client for interacting with remote caches in Bazel. */
65 | public class RemoteClient {
66 |
67 | private final AbstractRemoteActionCache cache;
68 | private final DigestUtil digestUtil;
69 |
70 | private RemoteClient(AbstractRemoteActionCache cache) {
71 | this.cache = cache;
72 | this.digestUtil = cache.getDigestUtil();
73 | }
74 |
75 | public AbstractRemoteActionCache getCache() {
76 | return cache;
77 | }
78 |
79 | // Prints the details (path and digest) of a DirectoryNode.
80 | private void printDirectoryNodeDetails(DirectoryNode directoryNode, Path directoryPath) {
81 | System.out.printf(
82 | "%s [Directory digest: %s]\n",
83 | directoryPath.toString(), digestUtil.toString(directoryNode.getDigest()));
84 | }
85 |
86 | // Prints the details (path and content digest) of a FileNode.
87 | private void printFileNodeDetails(FileNode fileNode, Path filePath) {
88 | System.out.printf(
89 | "%s [File content digest: %s]\n",
90 | filePath.toString(), digestUtil.toString(fileNode.getDigest()));
91 | }
92 |
93 | // List the files in a directory assuming the directory is at the given path. Returns the number
94 | // of files listed.
95 | private int listFileNodes(Path path, Directory dir, int limit) {
96 | int numFilesListed = 0;
97 | for (FileNode child : dir.getFilesList()) {
98 | if (numFilesListed >= limit) {
99 | System.out.println(" ... (too many files to list, some omitted)");
100 | break;
101 | }
102 | Path childPath = path.resolve(child.getName());
103 | printFileNodeDetails(child, childPath);
104 | numFilesListed++;
105 | }
106 | return numFilesListed;
107 | }
108 |
109 | // Recursively list directory files/subdirectories with digests. Returns the number of files
110 | // listed.
111 | private int listDirectory(Path path, Directory dir, Map childrenMap, int limit)
112 | throws IOException {
113 | // Try to list the files in this directory before listing the directories.
114 | int numFilesListed = listFileNodes(path, dir, limit);
115 | if (numFilesListed >= limit) {
116 | return numFilesListed;
117 | }
118 | for (DirectoryNode child : dir.getDirectoriesList()) {
119 | Path childPath = path.resolve(child.getName());
120 | printDirectoryNodeDetails(child, childPath);
121 | Digest childDigest = child.getDigest();
122 | Directory childDir = childrenMap.get(childDigest);
123 | numFilesListed += listDirectory(childPath, childDir, childrenMap, limit - numFilesListed);
124 | if (numFilesListed >= limit) {
125 | return numFilesListed;
126 | }
127 | }
128 | return numFilesListed;
129 | }
130 |
131 | // Recursively list OutputDirectory with digests.
132 | private void listOutputDirectory(OutputDirectory dir, int limit) throws IOException {
133 | Tree tree;
134 | try {
135 | tree = Tree.parseFrom(cache.downloadBlob(dir.getTreeDigest()));
136 | } catch (IOException e) {
137 | throw new IOException("Failed to obtain Tree for OutputDirectory.", e);
138 | }
139 | Map childrenMap = new HashMap<>();
140 | for (Directory child : tree.getChildrenList()) {
141 | childrenMap.put(digestUtil.compute(child), child);
142 | }
143 | System.out.printf("OutputDirectory rooted at %s:\n", dir.getPath());
144 | listDirectory(Paths.get(""), tree.getRoot(), childrenMap, limit);
145 | }
146 |
147 | // Recursively list directory files/subdirectories with digests given a Tree of the directory.
148 | private void listTree(Path path, Tree tree, int limit) throws IOException {
149 | Map childrenMap = new HashMap<>();
150 | for (Directory child : tree.getChildrenList()) {
151 | childrenMap.put(digestUtil.compute(child), child);
152 | }
153 | listDirectory(path, tree.getRoot(), childrenMap, limit);
154 | }
155 |
156 | private static int getNumFiles(Tree tree) {
157 | return tree.getChildrenList().stream().mapToInt(dir -> dir.getFilesCount()).sum();
158 | }
159 |
160 | // Outputs a bash executable line that corresponds to executing the given command.
161 | private static void printCommand(Command command) {
162 | for (EnvironmentVariable var : command.getEnvironmentVariablesList()) {
163 | System.out.printf("%s=%s \\\n", var.getName(), ShellEscaper.escapeString(var.getValue()));
164 | }
165 | System.out.print(" ");
166 |
167 | System.out.println(ShellEscaper.escapeJoinAll(command.getArgumentsList()));
168 | }
169 |
170 | private static void printList(List list, int limit) {
171 | if (list.isEmpty()) {
172 | System.out.println("(none)");
173 | return;
174 | }
175 | list.stream().limit(limit).forEach(name -> System.out.println(name));
176 | if (list.size() > limit) {
177 | System.out.println(" ... (too many to list, some omitted)");
178 | }
179 | }
180 |
181 | private Action getAction(Digest actionDigest) throws IOException {
182 | Action action;
183 | try {
184 | action = Action.parseFrom(cache.downloadBlob(actionDigest));
185 | } catch (IOException e) {
186 | throw new IOException("Could not obtain Action from digest.", e);
187 | }
188 | return action;
189 | }
190 |
191 | private Command getCommand(Digest commandDigest) throws IOException {
192 | Command command;
193 | try {
194 | command = Command.parseFrom(cache.downloadBlob(commandDigest));
195 | } catch (IOException e) {
196 | throw new IOException("Could not obtain Command from digest.", e);
197 | }
198 | return command;
199 | }
200 |
201 | // Output for print action command.
202 | private void printAction(Digest actionDigest, int limit) throws IOException {
203 | Action action = getAction(actionDigest);
204 | Command command = getCommand(action.getCommandDigest());
205 |
206 | System.out.printf("Command [digest: %s]:\n", digestUtil.toString(action.getCommandDigest()));
207 | printCommand(command);
208 |
209 | Tree tree = cache.getTree(action.getInputRootDigest());
210 | System.out.printf(
211 | "\nInput files [total: %d, root Directory digest: %s]:\n",
212 | getNumFiles(tree), digestUtil.toString(action.getInputRootDigest()));
213 | listTree(Paths.get(""), tree, limit);
214 |
215 | System.out.println("\nOutput files:");
216 | printList(command.getOutputFilesList(), limit);
217 |
218 | System.out.println("\nOutput directories:");
219 | printList(command.getOutputDirectoriesList(), limit);
220 |
221 | System.out.println("\nPlatform:");
222 | if (command.hasPlatform() && !command.getPlatform().getPropertiesList().isEmpty()) {
223 | System.out.println(command.getPlatform().toString());
224 | } else {
225 | System.out.println("(none)");
226 | }
227 | }
228 |
229 | // Display output file (either digest or raw bytes).
230 | private void printOutputFile(OutputFile file) {
231 | String contentString;
232 | if (file.hasDigest()) {
233 | contentString = "Content digest: " + digestUtil.toString(file.getDigest());
234 | } else {
235 | contentString = "No digest included. This likely indicates a server error.";
236 | }
237 | System.out.printf(
238 | "%s [%s, executable: %b]\n", file.getPath(), contentString, file.getIsExecutable());
239 | }
240 |
241 | // Output for print action result command.
242 | private void printActionResult(ActionResult result, int limit) throws IOException {
243 | System.out.println("Output files:");
244 | result.getOutputFilesList().stream().limit(limit).forEach(name -> printOutputFile(name));
245 | if (result.getOutputFilesList().size() > limit) {
246 | System.out.println(" ... (too many to list, some omitted)");
247 | } else if (result.getOutputFilesList().isEmpty()) {
248 | System.out.println("(none)");
249 | }
250 |
251 | System.out.println("\nOutput directories:");
252 | if (!result.getOutputDirectoriesList().isEmpty()) {
253 | for (OutputDirectory dir : result.getOutputDirectoriesList()) {
254 | listOutputDirectory(dir, limit);
255 | }
256 | } else {
257 | System.out.println("(none)");
258 | }
259 |
260 | System.out.println(String.format("\nExit code: %d", result.getExitCode()));
261 |
262 | System.out.println("\nStderr buffer:");
263 | if (result.hasStderrDigest()) {
264 | byte[] stderr = cache.downloadBlob(result.getStderrDigest());
265 | System.out.println(new String(stderr, UTF_8));
266 | } else {
267 | System.out.println(result.getStderrRaw().toStringUtf8());
268 | }
269 |
270 | System.out.println("\nStdout buffer:");
271 | if (result.hasStdoutDigest()) {
272 | byte[] stdout = cache.downloadBlob(result.getStdoutDigest());
273 | System.out.println(new String(stdout, UTF_8));
274 | } else {
275 | System.out.println(result.getStdoutRaw().toStringUtf8());
276 | }
277 | }
278 |
279 | // Given a docker run action, sets up a directory for an Action to be run in (download Action
280 | // inputs, set up output directories), and display a docker command that will run the Action.
281 | private void setupDocker(Action action, Path root) throws IOException {
282 | Command command = getCommand(action.getCommandDigest());
283 | setupDocker(command, action.getInputRootDigest(), root);
284 | }
285 |
286 | private void setupDocker(Command command, Digest inputRootDigest, Path root) throws IOException {
287 | System.out.printf("Setting up Action in directory %s...\n", root.toAbsolutePath());
288 |
289 | try {
290 | cache.downloadDirectory(root, inputRootDigest);
291 | } catch (IOException e) {
292 | throw new IOException("Failed to download action inputs.", e);
293 | }
294 |
295 | // Setup directory structure for outputs.
296 | for (String output : command.getOutputFilesList()) {
297 | Path file = root.resolve(output);
298 | if (java.nio.file.Files.exists(file)) {
299 | throw new FileSystemAlreadyExistsException("Output file already exists: " + file);
300 | }
301 | Files.createParentDirs(file.toFile());
302 | }
303 | for (String output : command.getOutputDirectoriesList()) {
304 | Path dir = root.resolve(output);
305 | if (java.nio.file.Files.exists(dir)) {
306 | throw new FileSystemAlreadyExistsException("Output directory already exists: " + dir);
307 | }
308 | java.nio.file.Files.createDirectories(dir);
309 | }
310 | DockerUtil util = new DockerUtil();
311 | String dockerCommand = util.getDockerCommand(command, root.toString());
312 | System.out.println("\nSuccessfully setup Action in directory " + root.toString() + ".");
313 | System.out.println("\nTo run the Action locally, run:");
314 | System.out.println(" " + dockerCommand);
315 | }
316 |
317 | private static RemoteClient makeClientWithOptions(
318 | RemoteOptions remoteOptions, AuthAndTLSOptions authAndTlsOptions) throws IOException {
319 | DigestUtil digestUtil = new DigestUtil(Hashing.sha256());
320 | AbstractRemoteActionCache cache;
321 |
322 | if (GrpcRemoteCache.isRemoteCacheOptions(remoteOptions)) {
323 | cache = new GrpcRemoteCache(remoteOptions, authAndTlsOptions, digestUtil);
324 | RequestMetadata metadata =
325 | RequestMetadata.newBuilder()
326 | .setToolDetails(ToolDetails.newBuilder().setToolName("remote_client"))
327 | .build();
328 | Context prevContext = TracingMetadataUtils.contextWithMetadata(metadata).attach();
329 | } else {
330 | throw new UnsupportedOperationException("Only gRPC remote cache supported currently.");
331 | }
332 | return new RemoteClient(cache);
333 | }
334 |
335 | private static void doPrintLog(String grpcLogFile, PrintLogCommand options) throws IOException {
336 | LogParserUtils parser = new LogParserUtils(grpcLogFile);
337 | parser.printLog(options);
338 | }
339 |
340 | private static void doFailedActions(String grpcLogFile, FailedActionsCommand options)
341 | throws IOException, ParamException {
342 | LogParserUtils parser = new LogParserUtils(grpcLogFile);
343 | parser.printFailedActions();
344 | }
345 |
346 | private static void doLs(LsCommand options, RemoteClient client) throws IOException {
347 | Tree tree = client.getCache().getTree(options.digest);
348 | client.listTree(Paths.get(""), tree, options.limit);
349 | }
350 |
351 | private static void doLsOutDir(LsOutDirCommand options, RemoteClient client) throws IOException {
352 | OutputDirectory dir;
353 | try {
354 | dir = OutputDirectory.parseFrom(client.getCache().downloadBlob(options.digest));
355 | } catch (IOException e) {
356 | throw new IOException("Failed to obtain OutputDirectory.", e);
357 | }
358 | client.listOutputDirectory(dir, options.limit);
359 | }
360 |
361 | private static void doGetDir(GetDirCommand options, RemoteClient client) throws IOException {
362 | client.getCache().downloadDirectory(options.path, options.digest);
363 | }
364 |
365 | private static void doGetOutDir(GetOutDirCommand options, RemoteClient client)
366 | throws IOException {
367 | OutputDirectory dir;
368 | try {
369 | dir = OutputDirectory.parseFrom(client.getCache().downloadBlob(options.digest));
370 | } catch (IOException e) {
371 | throw new IOException("Failed to obtain OutputDirectory.", e);
372 | }
373 | client.getCache().downloadOutputDirectory(dir, options.path);
374 | }
375 |
376 | private static void doCat(CatCommand options, RemoteClient client) throws IOException {
377 | OutputStream output;
378 | if (options.file != null) {
379 | output = new FileOutputStream(options.file);
380 |
381 | if (!options.file.exists()) {
382 | options.file.createNewFile();
383 | }
384 | } else {
385 | output = System.out;
386 | }
387 |
388 | try {
389 | client.getCache().downloadBlob(options.digest, output);
390 | } catch (CacheNotFoundException e) {
391 | System.err.println("Error: " + e);
392 | } finally {
393 | output.close();
394 | }
395 | }
396 |
397 | private static void doShowAction(ShowActionCommand options, RemoteClient client)
398 | throws IOException {
399 | client.printAction(options.actionDigest, options.limit);
400 | }
401 |
402 | private static void doShowActionResult(ShowActionResultCommand options, RemoteClient client)
403 | throws IOException {
404 | ActionResult.Builder builder = ActionResult.newBuilder();
405 | FileInputStream fin = new FileInputStream(options.file);
406 | TextFormat.getParser().merge(new InputStreamReader(fin), builder);
407 | client.printActionResult(builder.build(), options.limit);
408 | }
409 |
410 | private static void doRun(String grpcLogFile, RunCommand options, RemoteClient client)
411 | throws IOException, ParamException {
412 | Path path = options.path != null ? options.path : Files.createTempDir().toPath();
413 |
414 | if (options.actionDigest != null) {
415 | client.setupDocker(client.getAction(options.actionDigest), path);
416 | } else if (!grpcLogFile.isEmpty()) {
417 | LogParserUtils parser = new LogParserUtils(grpcLogFile);
418 | List actions = parser.failedActions();
419 | if (actions.size() == 0) {
420 | System.err.println("No action specified. No failed actions found in GRPC log.");
421 | System.exit(1);
422 | } else if (actions.size() > 1) {
423 | System.err.println(
424 | "No action specified. Multiple failed actions found in GRPC log. Add one of the following options:");
425 | for (Digest d : actions) {
426 | System.err.println(" --digest " + d.getHash() + "/" + d.getSizeBytes());
427 | }
428 | System.exit(1);
429 | }
430 | Digest action = actions.get(0);
431 | client.setupDocker(client.getAction(action), path);
432 | } else {
433 | System.err.println("Specify --action_digest or --grpc_log");
434 | System.exit(1);
435 | }
436 | }
437 |
438 | public static void main(String[] args) throws Exception {
439 | try {
440 | selectAndPerformCommand(args);
441 | } catch (io.grpc.StatusRuntimeException e) {
442 | Status s = Status.fromThrowable(e);
443 | if (s.getCode() == Status.Code.INTERNAL && s.getDescription().contains("http2")) {
444 | System.err.println("http2 exception. Did you forget --tls_enabled?");
445 | }
446 | throw e;
447 | }
448 | }
449 |
450 | public static void selectAndPerformCommand(String[] args) throws Exception {
451 | AuthAndTLSOptions authAndTlsOptions = new AuthAndTLSOptions();
452 | RemoteOptions remoteOptions = new RemoteOptions();
453 | RemoteClientOptions remoteClientOptions = new RemoteClientOptions();
454 | LsCommand lsCommand = new LsCommand();
455 | LsOutDirCommand lsOutDirCommand = new LsOutDirCommand();
456 | GetDirCommand getDirCommand = new GetDirCommand();
457 | GetOutDirCommand getOutDirCommand = new GetOutDirCommand();
458 | CatCommand catCommand = new CatCommand();
459 | FailedActionsCommand failedActionsCommand = new FailedActionsCommand();
460 | ShowActionCommand showActionCommand = new ShowActionCommand();
461 | ShowActionResultCommand showActionResultCommand = new ShowActionResultCommand();
462 | PrintLogCommand printLogCommand = new PrintLogCommand();
463 | RunCommand runCommand = new RunCommand();
464 |
465 | JCommander optionsParser =
466 | JCommander.newBuilder()
467 | .programName("remote_client")
468 | .addObject(authAndTlsOptions)
469 | .addObject(remoteOptions)
470 | .addObject(remoteClientOptions)
471 | .addCommand("ls", lsCommand)
472 | .addCommand("lsoutdir", lsOutDirCommand)
473 | .addCommand("getdir", getDirCommand)
474 | .addCommand("getoutdir", getOutDirCommand)
475 | .addCommand("cat", catCommand)
476 | .addCommand("show_action", showActionCommand, "sa")
477 | .addCommand("show_action_result", showActionResultCommand, "sar")
478 | .addCommand("printlog", printLogCommand)
479 | .addCommand("run", runCommand)
480 | .addCommand("failed_actions", failedActionsCommand)
481 | .build();
482 |
483 | try {
484 | optionsParser.parse(args);
485 | } catch (ParameterException e) {
486 | System.err.println("Unable to parse options: " + e.getLocalizedMessage());
487 | optionsParser.usage();
488 | System.exit(1);
489 | }
490 |
491 | if (remoteClientOptions.help) {
492 | optionsParser.usage();
493 | return;
494 | }
495 |
496 | if (optionsParser.getParsedCommand() == null) {
497 | System.err.println("No command specified.");
498 | optionsParser.usage();
499 | System.exit(1);
500 | }
501 |
502 | switch (optionsParser.getParsedCommand()) {
503 | case "printlog":
504 | doPrintLog(remoteClientOptions.grpcLog, printLogCommand);
505 | break;
506 | case "ls":
507 | doLs(lsCommand, makeClientWithOptions(remoteOptions, authAndTlsOptions));
508 | break;
509 | case "lsoutdir":
510 | doLsOutDir(lsOutDirCommand, makeClientWithOptions(remoteOptions, authAndTlsOptions));
511 | break;
512 | case "getdir":
513 | doGetDir(getDirCommand, makeClientWithOptions(remoteOptions, authAndTlsOptions));
514 | break;
515 | case "getoutdir":
516 | doGetOutDir(getOutDirCommand, makeClientWithOptions(remoteOptions, authAndTlsOptions));
517 | break;
518 | case "cat":
519 | doCat(catCommand, makeClientWithOptions(remoteOptions, authAndTlsOptions));
520 | break;
521 | case "show_action":
522 | doShowAction(showActionCommand, makeClientWithOptions(remoteOptions, authAndTlsOptions));
523 | break;
524 | case "show_action_result":
525 | doShowActionResult(
526 | showActionResultCommand, makeClientWithOptions(remoteOptions, authAndTlsOptions));
527 | break;
528 | case "run":
529 | doRun(
530 | remoteClientOptions.grpcLog,
531 | runCommand,
532 | makeClientWithOptions(remoteOptions, authAndTlsOptions));
533 | break;
534 | case "failed_actions":
535 | doFailedActions(remoteClientOptions.grpcLog, failedActionsCommand);
536 | break;
537 | default:
538 | throw new IllegalArgumentException("Unknown command.");
539 | }
540 | }
541 | }
542 |
--------------------------------------------------------------------------------
/src/main/java/com/google/devtools/build/remote/client/RemoteClientOptions.java:
--------------------------------------------------------------------------------
1 | // Copyright 2018 The Bazel Authors. All rights reserved.
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package com.google.devtools.build.remote.client;
16 |
17 | import build.bazel.remote.execution.v2.Digest;
18 | import com.beust.jcommander.IStringConverter;
19 | import com.beust.jcommander.Parameter;
20 | import com.beust.jcommander.ParameterException;
21 | import com.beust.jcommander.Parameters;
22 | import com.beust.jcommander.converters.FileConverter;
23 | import com.beust.jcommander.converters.PathConverter;
24 | import java.io.File;
25 | import java.nio.file.Path;
26 | import java.nio.file.Paths;
27 |
28 | /** Options for operation of a remote client. */
29 | @Parameters(separators = "=")
30 | public final class RemoteClientOptions {
31 | @Parameter(names = "--help", description = "This message.", help = true)
32 | public boolean help;
33 |
34 | @Parameter(names = "--grpc_log", description = "GRPC log to reference for additional information")
35 | public String grpcLog = "";
36 |
37 | @Parameters(
38 | commandDescription = "Recursively lists a Directory in remote cache.",
39 | separators = "=")
40 | public static class LsCommand {
41 | @Parameter(
42 | names = {"--digest", "-d"},
43 | required = true,
44 | converter = DigestConverter.class,
45 | description = "The digest of the Directory to list in hex_hash/size_bytes.")
46 | public Digest digest = null;
47 |
48 | @Parameter(
49 | names = {"--limit", "-l"},
50 | description = "The maximum number of files in the Directory to list.")
51 | public int limit = 100;
52 | }
53 |
54 | @Parameters(
55 | commandDescription = "Recursively lists an OutputDirectory in remote cache.",
56 | separators = "=")
57 | public static class LsOutDirCommand {
58 | @Parameter(
59 | names = {"--digest", "-d"},
60 | required = true,
61 | converter = DigestConverter.class,
62 | description = "The digest of the OutputDirectory to list in hex_hash/size_bytes.")
63 | public Digest digest = null;
64 |
65 | @Parameter(
66 | names = {"--limit", "-l"},
67 | description = "The maximum number of files in the OutputDirectory to list.")
68 | public int limit = 100;
69 | }
70 |
71 | @Parameters(
72 | commandDescription = "Recursively downloads a Directory from remote cache.",
73 | separators = "=")
74 | public static class GetDirCommand {
75 | @Parameter(
76 | names = {"--digest", "-d"},
77 | required = true,
78 | converter = DigestConverter.class,
79 | description = "The digest of the Directory to download in hex_hash/size_bytes.")
80 | public Digest digest = null;
81 |
82 | @Parameter(
83 | names = {"--path", "-o"},
84 | converter = PathConverter.class,
85 | description = "The local path to download the Directory contents into.")
86 | public Path path = Paths.get("");
87 | }
88 |
89 | @Parameters(
90 | commandDescription = "Recursively downloads a OutputDirectory from remote cache.",
91 | separators = "=")
92 | public static class GetOutDirCommand {
93 | @Parameter(
94 | names = {"--digest", "-d"},
95 | required = true,
96 | converter = DigestConverter.class,
97 | description = "The digest of the OutputDirectory to download in hex_hash/size_bytes.")
98 | public Digest digest = null;
99 |
100 | @Parameter(
101 | names = {"--path", "-o"},
102 | converter = PathConverter.class,
103 | description = "The local path to download the OutputDirectory contents into.")
104 | public Path path = Paths.get("");
105 | }
106 |
107 | @Parameters(
108 | commandDescription =
109 | "Write contents of a blob from remote cache to stdout. If specified, "
110 | + "the contents of the blob can be written to a specific file instead of stdout.",
111 | separators = "=")
112 | public static class CatCommand {
113 | @Parameter(
114 | names = {"--digest", "-d"},
115 | required = true,
116 | converter = DigestConverter.class,
117 | description = "The digest in the format hex_hash/size_bytes of the blob to download.")
118 | public Digest digest = null;
119 |
120 | @Parameter(
121 | names = {"--file", "-o"},
122 | converter = FileConverter.class,
123 | description = "Specifies a file to write the blob contents to instead of stdout.")
124 | public File file = null;
125 | }
126 |
127 | @Parameters(
128 | commandDescription = "Find and print action ids of failed actions from grpc log.",
129 | separators = "=")
130 | public static class FailedActionsCommand {}
131 |
132 | @Parameters(
133 | commandDescription = "Parse and display an Action with its corresponding command.",
134 | separators = "=")
135 | public static class ShowActionCommand {
136 | @Parameter(
137 | names = {"--textproto", "-p"},
138 | converter = FileConverter.class,
139 | description = "Path to a V1 Action proto stored in protobuf text format.")
140 | public File file = null;
141 |
142 | @Parameter(
143 | names = {"--digest", "-d"},
144 | converter = DigestConverter.class,
145 | description = "Action digest in the form hex_hash/size_bytes. Use for V2 API.")
146 | public Digest actionDigest = null;
147 |
148 | @Parameter(
149 | names = {"--limit", "-l"},
150 | description = "The maximum number of input/output files to list.")
151 | public int limit = 100;
152 | }
153 |
154 | @Parameters(commandDescription = "Parse and display an ActionResult.", separators = "=")
155 | public static class ShowActionResultCommand {
156 | @Parameter(
157 | names = {"--textproto", "-p"},
158 | required = true,
159 | converter = FileConverter.class,
160 | description = "Path to a ActionResult proto stored in protobuf text format.")
161 | public File file = null;
162 |
163 | @Parameter(
164 | names = {"--limit", "-l"},
165 | description = "The maximum number of output files to list.")
166 | public int limit = 100;
167 | }
168 |
169 | @Parameters(
170 | commandDescription =
171 | "Write all log entries from a Bazel gRPC log to standard output. The Bazel gRPC log "
172 | + "consists of a sequence of delimited serialized LogEntry protobufs, as produced by "
173 | + "the method LogEntry.writeDelimitedTo(OutputStream).",
174 | separators = "=")
175 | public static class PrintLogCommand {
176 | @Parameter(
177 | names = {"--group_by_action", "-g"},
178 | description =
179 | "Display entries grouped by action instead of individually. Entries are printed in order "
180 | + "of their call started timestamps (earliest first). Entries without action-id"
181 | + "metadata are skipped.")
182 | public boolean groupByAction;
183 |
184 | @Parameter(names = "--format_json", description = "Print the logs in json format")
185 | public boolean formatJson;
186 | }
187 |
188 | @Parameters(
189 | commandDescription =
190 | "Sets up a directory and Docker command to locally run a single action"
191 | + "given its Action proto. This requires the Action's inputs to be stored in CAS so that "
192 | + "they can be retrieved.",
193 | separators = "=")
194 | public static class RunCommand {
195 | @Parameter(
196 | names = {"--textproto", "-p"},
197 | converter = FileConverter.class,
198 | description =
199 | "Path to the Action proto stored in protobuf text format to be run in the "
200 | + "container.")
201 | public File file = null;
202 |
203 | @Parameter(
204 | names = {"--digest", "-d"},
205 | converter = DigestConverter.class,
206 | description = "Action digest in the form hex_hash/size_bytes. Use for V2 API.")
207 | public Digest actionDigest = null;
208 |
209 | @Parameter(
210 | names = {"--path", "-o"},
211 | converter = PathConverter.class,
212 | description = "Path to set up the action inputs in.")
213 | public Path path = null;
214 | }
215 |
216 | /** Converter for hex_hash/size_bytes string to a Digest object. */
217 | public static class DigestConverter implements IStringConverter {
218 | @Override
219 | public Digest convert(String input) {
220 | int slash = input.indexOf('/');
221 | if (slash < 0) {
222 | throw new ParameterException("'" + input + "' is not as hex_hash/size_bytes");
223 | }
224 | try {
225 | long size = Long.parseLong(input.substring(slash + 1));
226 | return DigestUtil.buildDigest(input.substring(0, slash), size);
227 | } catch (NumberFormatException e) {
228 | throw new ParameterException("'" + input + "' is not a hex_hash/size_bytes: " + e);
229 | }
230 | }
231 | }
232 | }
233 |
--------------------------------------------------------------------------------
/src/main/java/com/google/devtools/build/remote/client/RemoteOptions.java:
--------------------------------------------------------------------------------
1 | // Copyright 2018 The Bazel Authors. All rights reserved.
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package com.google.devtools.build.remote.client;
16 |
17 | import com.beust.jcommander.DynamicParameter;
18 | import com.beust.jcommander.Parameter;
19 | import com.beust.jcommander.Parameters;
20 | import java.util.HashMap;
21 | import java.util.Map;
22 |
23 | /** Options for remote cache. */
24 | @Parameters(separators = "=")
25 | public final class RemoteOptions {
26 | @Parameter(
27 | names = "--remote_http_cache",
28 | description =
29 | "A base URL of a HTTP caching service. Both http:// and https:// are supported. BLOBs are "
30 | + "are stored with PUT and retrieved with GET. See remote/README.md for more"
31 | + "information.")
32 | public String remoteHttpCache = null;
33 |
34 | @Parameter(
35 | names = "--remote_cache",
36 | description = "HOST or HOST:PORT of a remote caching endpoint.")
37 | public String remoteCache = null;
38 |
39 | @Parameter(
40 | names = "--remote_timeout",
41 | description = "The maximum number of seconds to wait for remote execution and cache calls.")
42 | public int remoteTimeout = 60;
43 |
44 | @Parameter(
45 | names = "--remote_instance_name",
46 | description = "Value to pass as instance_name in the remote execution API.")
47 | public String remoteInstanceName = "";
48 |
49 | @DynamicParameter(
50 | names = "--remote_header",
51 | description = "Headers to be passed to the remote cache. Use multiple times for multiple headers.")
52 | public Map remoteHeaders = new HashMap<>();
53 | }
54 |
--------------------------------------------------------------------------------
/src/main/java/com/google/devtools/build/remote/client/ShellEscaper.java:
--------------------------------------------------------------------------------
1 | // Copyright 2018 The Bazel Authors. All rights reserved.
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package com.google.devtools.build.remote.client;
16 |
17 | import com.google.common.base.CharMatcher;
18 | import com.google.common.base.Function;
19 | import com.google.common.base.Joiner;
20 | import com.google.common.collect.Iterables;
21 | import com.google.common.escape.CharEscaperBuilder;
22 | import com.google.common.escape.Escaper;
23 | import java.util.List;
24 |
25 | /**
26 | * Utility class to escape strings for use with shell commands.
27 | *
28 | * The code for this class was based on
29 | * src/main/java/com/google/devtools/build/lib/util/ShellEscaper.java from the Bazel codebase.
30 | *
31 | *
Escaped strings may safely be inserted into shell commands. Escaping is only done if
32 | * necessary. Strings containing only shell-neutral characters will not be escaped.
33 | */
34 | public class ShellEscaper extends Escaper {
35 | public static final ShellEscaper INSTANCE = new ShellEscaper();
36 |
37 | private static final Function AS_FUNCTION = INSTANCE.asFunction();
38 |
39 | private static final Joiner SPACE_JOINER = Joiner.on(' ');
40 | private static final Escaper STRONGQUOTE_ESCAPER =
41 | new CharEscaperBuilder().addEscape('\'', "'\\''").toEscaper();
42 | private static final CharMatcher SAFECHAR_MATCHER =
43 | CharMatcher.anyOf("@%-_+:,./")
44 | .or(CharMatcher.inRange('0', '9')) // We can't use CharMatcher.javaLetterOrDigit(),
45 | .or(CharMatcher.inRange('a', 'z')) // that would also accept non-ASCII digits and
46 | .or(CharMatcher.inRange('A', 'Z')) // letters.
47 | .precomputed();
48 |
49 | /**
50 | * Escape an argument so that it can passed as a single argument in bash command line. Unless the
51 | * argument contains no special characters, it will be wrapped in single quotes to escape special
52 | * behaviour. In the case that the arguments itself has single quotes, the inner single quotes are
53 | * escaped as a special case to avoid conflicting with the out single quotes.
54 | */
55 | public String escape(String unescaped) {
56 | final String s = unescaped.toString();
57 | if (s.isEmpty()) {
58 | // Empty string is a special case: needs to be quoted to ensure that it
59 | // gets treated as a separate argument.
60 | return "''";
61 | } else {
62 | return SAFECHAR_MATCHER.matchesAllOf(s) ? s : "'" + STRONGQUOTE_ESCAPER.escape(s) + "'";
63 | }
64 | }
65 |
66 | public static String escapeString(String unescaped) {
67 | return INSTANCE.escape(unescaped);
68 | }
69 |
70 | /**
71 | * Returns a string containing the argument strings in the given list escaped and joined by
72 | * spaces.
73 | */
74 | public static String escapeJoinAll(List args) {
75 | return SPACE_JOINER.join(Iterables.transform(args, AS_FUNCTION));
76 | }
77 | }
78 |
--------------------------------------------------------------------------------
/src/main/java/com/google/devtools/build/remote/client/TracingMetadataUtils.java:
--------------------------------------------------------------------------------
1 | // Copyright 2018 The Bazel Authors. All rights reserved.
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | package com.google.devtools.build.remote.client;
15 |
16 | import build.bazel.remote.execution.v2.RequestMetadata;
17 | import com.google.common.annotations.VisibleForTesting;
18 | import io.grpc.ClientInterceptor;
19 | import io.grpc.Context;
20 | import io.grpc.Metadata;
21 | import io.grpc.protobuf.ProtoUtils;
22 | import io.grpc.stub.MetadataUtils;
23 |
24 | /** Utility functions to handle Metadata for remote Grpc calls. */
25 | public class TracingMetadataUtils {
26 |
27 | private TracingMetadataUtils() {}
28 |
29 | private static final Context.Key CONTEXT_KEY =
30 | Context.key("remote-grpc-metadata");
31 |
32 | @VisibleForTesting
33 | public static final Metadata.Key METADATA_KEY =
34 | ProtoUtils.keyForProto(RequestMetadata.getDefaultInstance());
35 |
36 | /**
37 | * Returns a new gRPC context derived from the current context, with {@link RequestMetadata}
38 | * accessible by the {@link fromCurrentContext()} method.
39 | */
40 | public static Context contextWithMetadata(RequestMetadata metadata) {
41 | return Context.current().withValue(CONTEXT_KEY, metadata);
42 | }
43 |
44 | /**
45 | * Fetches a {@link RequestMetadata} defined on the current context.
46 | *
47 | * @throws {@link IllegalStateException} when the metadata is not defined in the current context.
48 | */
49 | public static RequestMetadata fromCurrentContext() {
50 | RequestMetadata metadata = CONTEXT_KEY.get();
51 | if (metadata == null) {
52 | throw new IllegalStateException("RequestMetadata not set in current context.");
53 | }
54 | return metadata;
55 | }
56 |
57 | /**
58 | * Creates a {@link Metadata} containing the {@link RequestMetadata} defined on the current
59 | * context.
60 | *
61 | * @throws {@link IllegalStateException} when the metadata is not defined in the current context.
62 | */
63 | public static Metadata headersFromCurrentContext() {
64 | Metadata headers = new Metadata();
65 | headers.put(METADATA_KEY, fromCurrentContext());
66 | return headers;
67 | }
68 |
69 | public static ClientInterceptor attachMetadataFromContextInterceptor() {
70 | return MetadataUtils.newAttachHeadersInterceptor(headersFromCurrentContext());
71 | }
72 | }
73 |
--------------------------------------------------------------------------------
/src/main/proto/BUILD:
--------------------------------------------------------------------------------
1 | package(default_visibility = ["//visibility:public"])
2 |
3 | proto_library(
4 | name = "remote_execution_log_proto",
5 | srcs = ["remote_execution_log.proto"],
6 | deps = [
7 | "@com_google_protobuf//:timestamp_proto",
8 | "@googleapis//google/bytestream:bytestream_proto",
9 | "@googleapis//google/longrunning:operations_proto",
10 | "@googleapis//google/rpc:status_proto",
11 | "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_proto",
12 | ],
13 | )
14 |
15 | java_proto_library(
16 | name = "remote_execution_log_java_proto",
17 | deps = [":remote_execution_log_proto"],
18 | )
19 |
--------------------------------------------------------------------------------
/src/main/proto/remote_execution_log.proto:
--------------------------------------------------------------------------------
1 | // Copyright 2018 The Bazel Authors. All rights reserved.
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | syntax = "proto3";
16 |
17 | package remote_logging;
18 |
19 | import "build/bazel/remote/execution/v2/remote_execution.proto";
20 | import "google/protobuf/timestamp.proto";
21 | import "google/bytestream/bytestream.proto";
22 | import "google/longrunning/operations.proto";
23 | import "google/rpc/status.proto";
24 |
25 | option java_package = "com.google.devtools.build.lib.remote.logging";
26 |
27 | // A single log entry for gRPC calls related to remote execution.
28 | message LogEntry {
29 | // Request metadata included in call.
30 | build.bazel.remote.execution.v2.RequestMetadata metadata = 1;
31 |
32 | // Status of the call on close.
33 | google.rpc.Status status = 2;
34 |
35 | // Full method name of the method called as returned from
36 | // io.grpc.MethodDescriptor.getFullMethodName() (i.e. in format
37 | // $FULL_SERVICE_NAME/$METHOD_NAME).
38 | string method_name = 3;
39 |
40 | // Method specific details for this call.
41 | RpcCallDetails details = 4;
42 |
43 | // Time the call started.
44 | google.protobuf.Timestamp start_time = 5;
45 |
46 | // Time the call closed.
47 | google.protobuf.Timestamp end_time = 6;
48 | }
49 |
50 | // Details for a call to
51 | // build.bazel.remote.execution.v2.Execution.Execute.
52 | message ExecuteDetails {
53 | // The build.bazel.remote.execution.v2.ExecuteRequest sent by the
54 | // call.
55 | build.bazel.remote.execution.v2.ExecuteRequest request = 1;
56 |
57 | // Each google.longrunning.Operation received by the Execute call in order.
58 | repeated google.longrunning.Operation responses = 2;
59 | }
60 |
61 | // Details for a call to
62 | // build.bazel.remote.execution.v2.ActionCache.GetCapabilities.
63 | message GetCapabilitiesDetails {
64 | // The build.bazel.remote.execution.v2.GetCapabilitiesRequest sent by
65 | // the call.
66 | build.bazel.remote.execution.v2.GetCapabilitiesRequest request = 1;
67 |
68 | // The received build.bazel.remote.execution.v2.ServerCapabilities.
69 | build.bazel.remote.execution.v2.ServerCapabilities response = 2;
70 | }
71 |
72 | // Details for a call to
73 | // build.bazel.remote.execution.v2.ActionCache.GetActionResult.
74 | message GetActionResultDetails {
75 | // The build.bazel.remote.execution.v2.GetActionResultRequest sent by
76 | // the call.
77 | build.bazel.remote.execution.v2.GetActionResultRequest request = 1;
78 |
79 | // The received build.bazel.remote.execution.v2.ActionResult.
80 | build.bazel.remote.execution.v2.ActionResult response = 2;
81 | }
82 |
83 | // Details for a call to
84 | // build.bazel.remote.execution.v2.ActionCache.UpdateActionResult.
85 | message UpdateActionResultDetails {
86 | // The build.bazel.remote.execution.v2.GetActionResultRequest sent by
87 | // the call.
88 | build.bazel.remote.execution.v2.UpdateActionResultRequest request = 1;
89 |
90 | // The received build.bazel.remote.execution.v2.ActionResult.
91 | build.bazel.remote.execution.v2.ActionResult response = 2;
92 | }
93 |
94 | // Details for a call to build.bazel.remote.execution.v2.WaitExecution.
95 | message WaitExecutionDetails {
96 | // The google.watcher.v1.Request sent by the Watch call.
97 | build.bazel.remote.execution.v2.WaitExecutionRequest request = 1;
98 |
99 | // Each google.longrunning.Operation received by the call in order.
100 | repeated google.longrunning.Operation responses = 2;
101 | }
102 |
103 | // Details for a call to
104 | // build.bazel.remote.execution.v2.ContentAddressableStorage.FindMissingBlobs.
105 | message FindMissingBlobsDetails {
106 | // The build.bazel.remote.execution.v2.FindMissingBlobsRequest request
107 | // sent.
108 | build.bazel.remote.execution.v2.FindMissingBlobsRequest request = 1;
109 |
110 | // The build.bazel.remote.execution.v2.FindMissingBlobsResponse
111 | // received.
112 | build.bazel.remote.execution.v2.FindMissingBlobsResponse response = 2;
113 | }
114 |
115 | // Details for a call to google.bytestream.Read.
116 | message ReadDetails {
117 | // The google.bytestream.ReadRequest sent.
118 | google.bytestream.ReadRequest request = 1;
119 |
120 | // The number of reads performed in this call.
121 | int64 num_reads = 2;
122 |
123 | // The total number of bytes read totalled over all stream responses.
124 | int64 bytes_read = 3;
125 | }
126 |
127 | // Details for a call to google.bytestream.Write.
128 | message WriteDetails {
129 | // The names of resources requested to be written to in this call in the order
130 | // they were first requested in. If the ByteStream protocol is followed
131 | // according to specification, this should contain at most two elements:
132 | // The resource name specified in the first message of the stream, and an
133 | // empty string specified in each successive request if num_writes > 1.
134 | repeated string resource_names = 1;
135 |
136 | // The offsets sent for the initial request and any non-sequential offsets
137 | // specified over the course of the call. If the ByteStream protocol is
138 | // followed according to specification, this should contain a single element
139 | // which is the starting point for the write call.
140 | repeated int64 offsets = 5;
141 |
142 | // The effective final size for each request sent with finish_write true
143 | // specified over the course of the call. If the ByteStream protocol is
144 | // followed according to specification, this should contain a single element
145 | // which is the total size of the written resource, including the initial
146 | // offset.
147 | repeated int64 finish_writes = 6;
148 |
149 | // The number of writes performed in this call.
150 | int64 num_writes = 2;
151 |
152 | // The total number of bytes sent over the stream.
153 | int64 bytes_sent = 3;
154 |
155 | // The received google.bytestream.WriteResponse.
156 | google.bytestream.WriteResponse response = 4;
157 | }
158 |
159 | // Details for a call to google.bytestream.QueryWriteStatus.
160 | message QueryWriteStatusDetails {
161 | // The google.bytestream.QueryWriteStatusRequest sent by the call.
162 | google.bytestream.QueryWriteStatusRequest request = 1;
163 |
164 | // The received google.bytestream.QueryWriteStatusResponse.
165 | google.bytestream.QueryWriteStatusResponse response = 2;
166 | }
167 |
168 | // Contains details for specific types of calls.
169 | message RpcCallDetails {
170 | // For now this is kept backwards compabile with v1 version of API.
171 | // The calls that are different between the two APIs have different
172 | // field numbers: e.g., the log created by Bazel 16 will have v1_execute
173 | // field populated whereas the log created by Bazel 17 will have execute
174 | // field populated.
175 | oneof details {
176 | ExecuteDetails execute = 7;
177 | GetActionResultDetails get_action_result = 8;
178 | WaitExecutionDetails wait_execution = 9;
179 | FindMissingBlobsDetails find_missing_blobs = 10;
180 | ReadDetails read = 5;
181 | WriteDetails write = 6;
182 | GetCapabilitiesDetails get_capabilities = 12;
183 | UpdateActionResultDetails update_action_result = 13;
184 | QueryWriteStatusDetails query_write_status = 14;
185 | }
186 | }
187 |
--------------------------------------------------------------------------------
/src/test/java/com/google/devtools/build/remote/client/ActionGroupingTest.java:
--------------------------------------------------------------------------------
1 | // Copyright 2018 The Bazel Authors. All rights reserved.
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | package com.google.devtools.build.remote.client;
15 |
16 | import static com.google.common.truth.Truth.assertThat;
17 | import static com.google.common.truth.Truth.assertWithMessage;
18 |
19 | import build.bazel.remote.execution.v2.ActionResult;
20 | import build.bazel.remote.execution.v2.Digest;
21 | import build.bazel.remote.execution.v2.ExecuteResponse;
22 | import build.bazel.remote.execution.v2.GetActionResultRequest;
23 | import build.bazel.remote.execution.v2.RequestMetadata;
24 | import com.google.devtools.build.lib.remote.logging.RemoteExecutionLog.ExecuteDetails;
25 | import com.google.devtools.build.lib.remote.logging.RemoteExecutionLog.GetActionResultDetails;
26 | import com.google.devtools.build.lib.remote.logging.RemoteExecutionLog.LogEntry;
27 | import com.google.devtools.build.lib.remote.logging.RemoteExecutionLog.RpcCallDetails;
28 | import com.google.devtools.build.lib.remote.logging.RemoteExecutionLog.WaitExecutionDetails;
29 | import com.google.devtools.build.remote.client.ActionGrouping.ActionDetails;
30 | import com.google.devtools.build.remote.client.LogParserUtils.ParamException;
31 | import com.google.longrunning.Operation;
32 | import com.google.protobuf.Any;
33 | import com.google.protobuf.util.Timestamps;
34 | import io.grpc.Status;
35 | import io.grpc.Status.Code;
36 | import java.io.ByteArrayOutputStream;
37 | import java.io.IOException;
38 | import java.io.PrintStream;
39 | import java.io.PrintWriter;
40 | import java.io.StringWriter;
41 | import java.util.Arrays;
42 | import java.util.List;
43 | import java.util.Scanner;
44 | import org.junit.Test;
45 | import org.junit.runner.RunWith;
46 | import org.junit.runners.JUnit4;
47 |
48 | /** Tests for {@link ShellEscaper}. */
49 | @RunWith(JUnit4.class)
50 | public class ActionGroupingTest {
51 |
52 | /** Returns the string obtained from printByAction in actionGrouping */
53 | private String getOutput(ActionGrouping actionGrouping) throws IOException {
54 | StringWriter stringOut = new StringWriter();
55 | PrintWriter printWriter = new PrintWriter(stringOut);
56 |
57 | actionGrouping.printByAction(printWriter);
58 | printWriter.flush();
59 | return stringOut.toString();
60 | }
61 | /*
62 | * Asserts that the actionGrouping contains the given strings.
63 | * To pass, all action and entry delimiters must be explicitly included in the arguments
64 | * (this way an extra action or an entry would fail the test).
65 | * All other arguments need to be substrings of a single line.
66 | */
67 | private void checkOutput(ActionGrouping actionGrouping, String... args) throws IOException {
68 | String result = getOutput(actionGrouping);
69 |
70 | // A dummy string to use when we run out of args.
71 | // We don't expect it to appear in the input
72 | final String sentinel = "<<<<>>>> No more inputs to process <<<<>>>>";
73 |
74 | Scanner scanner = new Scanner(result);
75 | int ind = 0;
76 | int lineNum = 0;
77 | while (scanner.hasNextLine()) {
78 | String got = scanner.nextLine().trim();
79 | lineNum++;
80 | String want = (ind < args.length ? args[ind] : sentinel);
81 | if (got.contains(want)) {
82 | ind++;
83 | continue;
84 | }
85 |
86 | assertWithMessage(
87 | "Expecting "
88 | + want
89 | + ", got action delimiter on line "
90 | + lineNum
91 | + ".Output: \n"
92 | + result)
93 | .that(got)
94 | .isNotEqualTo(ActionGrouping.actionDelimiter);
95 |
96 | assertWithMessage(
97 | "Expecting "
98 | + want
99 | + ", got entry delimiter on line "
100 | + lineNum
101 | + ".Output: \n"
102 | + result)
103 | .that(got)
104 | .isNotEqualTo(ActionGrouping.entryDelimiter);
105 |
106 | // Ignore unmached lines that are not delimiters
107 | }
108 | assertWithMessage(
109 | "Not all expected arguments are found. "
110 | + (ind < args.length ? "Looking for " + args[ind] : "Expected no output")
111 | + ". Output: \n"
112 | + result)
113 | .that(ind)
114 | .isEqualTo(args.length);
115 | scanner.close();
116 | }
117 |
118 | @Test
119 | public void EmptyGrouping() throws Exception {
120 | ActionGrouping.Builder actionGrouping = new ActionGrouping.Builder();
121 | checkOutput(actionGrouping.build()); // This will ensure there are no delimiters in the output
122 | }
123 |
124 | private LogEntry getLogEntry(String actionId, String method, int nanos, RpcCallDetails details) {
125 | RequestMetadata m = RequestMetadata.newBuilder().setActionId(actionId).build();
126 | LogEntry.Builder result =
127 | LogEntry.newBuilder()
128 | .setMetadata(m)
129 | .setMethodName(method)
130 | .setStartTime(Timestamps.fromNanos(nanos));
131 | if (details != null) {
132 | result.setDetails(details);
133 | }
134 | return result.build();
135 | }
136 |
137 | private LogEntry getLogEntry(String actionId, String method, int nanos) {
138 | return getLogEntry(actionId, method, nanos, null);
139 | }
140 |
141 | private String actionHeader(String actionId) {
142 | return String.format(ActionGrouping.actionString, actionId).trim();
143 | }
144 |
145 | @Test
146 | public void SingleLog() throws Exception {
147 | ActionGrouping.Builder actionGrouping = new ActionGrouping.Builder();
148 | actionGrouping.addLogEntry(getLogEntry("action1", "call1", 4));
149 | checkOutput(
150 | actionGrouping.build(),
151 | ActionGrouping.actionDelimiter,
152 | actionHeader("action1"),
153 | ActionGrouping.actionDelimiter,
154 | "call1",
155 | ActionGrouping.entryDelimiter);
156 | }
157 |
158 | @Test
159 | public void SameTimestamp() throws Exception {
160 | // Events with the same timestamp must all be present, but their ordering is arbitrary.
161 | ActionGrouping.Builder actionGrouping = new ActionGrouping.Builder();
162 | actionGrouping.addLogEntry(getLogEntry("action1", "a1_call1", 5));
163 | actionGrouping.addLogEntry(getLogEntry("action2", "a2_call1", 5));
164 | actionGrouping.addLogEntry(getLogEntry("action1", "a1_call2", 5));
165 | actionGrouping.addLogEntry(getLogEntry("action1", "a1_call3", 5));
166 |
167 | String result = getOutput(actionGrouping.build());
168 |
169 | assertWithMessage("Output: \n" + result).that(result).contains("a1_call1");
170 | assertWithMessage("Output: \n" + result).that(result).contains("a1_call2");
171 | assertWithMessage("Output: \n" + result).that(result).contains("a1_call3");
172 | assertWithMessage("Output: \n" + result).that(result).contains("a2_call1");
173 | }
174 |
175 | @Test
176 | public void Sorting() throws Exception {
177 | ActionGrouping.Builder actionGrouping = new ActionGrouping.Builder();
178 | actionGrouping.addLogEntry(getLogEntry("action1", "a1_call_5", 5));
179 | actionGrouping.addLogEntry(getLogEntry("action2", "a2_call_10", 10));
180 | actionGrouping.addLogEntry(getLogEntry("action1", "a1_call_3", 3));
181 | actionGrouping.addLogEntry(getLogEntry("action1", "a1_call_1", 1));
182 | actionGrouping.addLogEntry(getLogEntry("action1", "a1_call_100", 100));
183 | actionGrouping.addLogEntry(getLogEntry("action2", "a2_call_100", 100));
184 | actionGrouping.addLogEntry(getLogEntry("action2", "a2_call_50", 50));
185 | actionGrouping.addLogEntry(getLogEntry("action3", "a3_call_1", 1));
186 | checkOutput(
187 | actionGrouping.build(),
188 | ActionGrouping.actionDelimiter,
189 | actionHeader("action1"),
190 | ActionGrouping.actionDelimiter,
191 | "a1_call_1",
192 | ActionGrouping.entryDelimiter,
193 | "a1_call_3",
194 | ActionGrouping.entryDelimiter,
195 | "a1_call_5",
196 | ActionGrouping.entryDelimiter,
197 | "a1_call_100",
198 | ActionGrouping.entryDelimiter,
199 | ActionGrouping.actionDelimiter,
200 | actionHeader("action2"),
201 | ActionGrouping.actionDelimiter,
202 | "a2_call_10",
203 | ActionGrouping.entryDelimiter,
204 | "a2_call_50",
205 | ActionGrouping.entryDelimiter,
206 | "a2_call_100",
207 | ActionGrouping.entryDelimiter,
208 | ActionGrouping.actionDelimiter,
209 | actionHeader("action3"),
210 | ActionGrouping.actionDelimiter,
211 | "a3_call_1",
212 | ActionGrouping.entryDelimiter);
213 | }
214 |
215 | ActionResult actionResultSuccess = ActionResult.newBuilder().setExitCode(0).build();
216 | ActionResult actionResultFail = ActionResult.newBuilder().setExitCode(1).build();
217 |
218 | Digest toDigest(String d) {
219 | String[] parts = d.split("/");
220 | assert (parts.length == 2);
221 | long size = Long.parseLong(parts[1]);
222 | return Digest.newBuilder().setHash(parts[0]).setSizeBytes(size).build();
223 | }
224 |
225 | private LogEntry makeFailedGetActionResult(int nanos, String digest) {
226 | RpcCallDetails.Builder details = RpcCallDetails.newBuilder();
227 | details.getGetActionResultBuilder().getRequestBuilder().setActionDigest(toDigest(digest));
228 | LogEntry logEntry =
229 | getLogEntry(toDigest(digest).getHash(), "getActionResult", nanos, details.build());
230 | LogEntry.Builder result = LogEntry.newBuilder(logEntry);
231 | result.getStatusBuilder().setCode(Status.NOT_FOUND.getCode().value());
232 | return result.build();
233 | }
234 |
235 | private RpcCallDetails makeGetActionResult(ActionResult result) {
236 | GetActionResultDetails getActionResult =
237 | GetActionResultDetails.newBuilder().setResponse(result).build();
238 | return RpcCallDetails.newBuilder().setGetActionResult(getActionResult).build();
239 | }
240 |
241 | private RpcCallDetails makeGetActionResultWithDigest(ActionResult result, String digest) {
242 | GetActionResultRequest request =
243 | GetActionResultRequest.newBuilder().setActionDigest(toDigest(digest)).build();
244 | GetActionResultDetails getActionResult =
245 | GetActionResultDetails.newBuilder().setResponse(result).build();
246 | return RpcCallDetails.newBuilder().setGetActionResult(getActionResult).build();
247 | }
248 |
249 | private RpcCallDetails makeExecute(ActionResult result) {
250 | ExecuteResponse response = ExecuteResponse.newBuilder().setResult(result).build();
251 | Operation operation =
252 | Operation.newBuilder().setResponse(Any.pack(response)).setDone(true).build();
253 | ExecuteDetails execute = ExecuteDetails.newBuilder().addResponses(operation).build();
254 | return RpcCallDetails.newBuilder().setExecute(execute).build();
255 | }
256 |
257 | private RpcCallDetails makeExecuteWithStatus(int status) {
258 | ExecuteResponse.Builder response = ExecuteResponse.newBuilder();
259 | response.getStatusBuilder().setCode(status);
260 | Operation operation =
261 | Operation.newBuilder().setResponse(Any.pack(response.build())).setDone(true).build();
262 | ExecuteDetails execute = ExecuteDetails.newBuilder().addResponses(operation).build();
263 | return RpcCallDetails.newBuilder().setExecute(execute).build();
264 | }
265 |
266 | private RpcCallDetails makeWatch(ActionResult result) {
267 | ExecuteResponse response = ExecuteResponse.newBuilder().setResult(result).build();
268 | Operation operation =
269 | Operation.newBuilder().setResponse(Any.pack(response)).setDone(true).build();
270 | WaitExecutionDetails waitExecution =
271 | WaitExecutionDetails.newBuilder().addResponses(operation).build();
272 | return RpcCallDetails.newBuilder().setWaitExecution(waitExecution).build();
273 | }
274 |
275 | private LogEntry addDigest(LogEntry entry, String digest) {
276 | LogEntry.Builder result = LogEntry.newBuilder(entry);
277 | assert (entry.hasDetails());
278 | if (entry.getDetails().hasExecute()) {
279 | result
280 | .getDetailsBuilder()
281 | .getExecuteBuilder()
282 | .getRequestBuilder()
283 | .setActionDigest(toDigest(digest));
284 | } else if (entry.getDetails().hasGetActionResult()) {
285 | result
286 | .getDetailsBuilder()
287 | .getGetActionResultBuilder()
288 | .getRequestBuilder()
289 | .setActionDigest(toDigest(digest));
290 | } else {
291 | assertWithMessage("Can't add digest to an entry that is neither Execute nor GetActionResult")
292 | .fail();
293 | }
294 | return result.build();
295 | }
296 |
297 | // Test logic to extract action result
298 | @Test
299 | public void ActionResultForEmpty() {
300 | ActionDetails.Builder detailsBuilder = new ActionDetails.Builder("actionId");
301 | ActionDetails details = detailsBuilder.build();
302 | // Action with no log entries is not failed but has no executeResponse
303 | assert (details.getExecuteResponse() == null);
304 | assert (!details.isFailed());
305 | };
306 |
307 | @Test
308 | public void ActionResultFromCachePass() throws IOException {
309 | ActionDetails.Builder detailsBuilder = new ActionDetails.Builder("actionId");
310 | detailsBuilder.add(
311 | getLogEntry("actionId", "Execute", 10, makeGetActionResult(actionResultSuccess)));
312 | ActionDetails details = detailsBuilder.build();
313 | assert (details.getExecuteResponse() != null);
314 | assert (!details.isFailed());
315 | };
316 |
317 | @Test
318 | public void ActionResultFromCacheFail() throws IOException {
319 | ActionDetails.Builder detailsBuilder = new ActionDetails.Builder("actionId");
320 | detailsBuilder.add(
321 | getLogEntry("actionId", "Execute", 10, makeGetActionResult(actionResultFail)));
322 | ActionDetails details = detailsBuilder.build();
323 | assert (details.getExecuteResponse() != null);
324 | assert (details.isFailed());
325 | };
326 |
327 | @Test
328 | public void ActionResultFromExecutePass() throws IOException {
329 | ActionDetails.Builder detailsBuilder = new ActionDetails.Builder("actionId");
330 | detailsBuilder.add(getLogEntry("actionId", "Execute", 10, makeExecute(actionResultSuccess)));
331 | ActionDetails details = detailsBuilder.build();
332 | assert (details.getExecuteResponse() != null);
333 | assert (!details.isFailed());
334 | };
335 |
336 | @Test
337 | public void ActionResultFromExecuteFail() throws IOException {
338 | ActionDetails.Builder detailsBuilder = new ActionDetails.Builder("actionId");
339 | detailsBuilder.add(getLogEntry("actionId", "Execute", 10, makeExecute(actionResultFail)));
340 | ActionDetails details = detailsBuilder.build();
341 | assert (details.getExecuteResponse() != null);
342 | assert (details.isFailed());
343 | };
344 |
345 | @Test
346 | public void ActionResultFromExecuteError() throws IOException {
347 | ActionDetails.Builder detailsBuilder = new ActionDetails.Builder("actionId");
348 | detailsBuilder.add(
349 | getLogEntry(
350 | "actionId", "Execute", 10, makeExecuteWithStatus(Code.DEADLINE_EXCEEDED.value())));
351 | ActionDetails details = detailsBuilder.build();
352 | assert (details.getExecuteResponse() != null);
353 | assert (details.isFailed());
354 | };
355 |
356 | @Test
357 | public void ActionResultFromWatchPass() throws IOException {
358 | ActionDetails.Builder detailsBuilder = new ActionDetails.Builder("actionId");
359 | detailsBuilder.add(getLogEntry("actionId", "Execute", 10, makeWatch(actionResultSuccess)));
360 | ActionDetails details = detailsBuilder.build();
361 | assert (details.getExecuteResponse() != null);
362 | assert (!details.isFailed());
363 | };
364 |
365 | @Test
366 | public void ActionResultFromWatchFail() throws IOException {
367 | ActionDetails.Builder detailsBuilder = new ActionDetails.Builder("actionId");
368 | detailsBuilder.add(getLogEntry("actionId", "Execute", 10, makeWatch(actionResultFail)));
369 | ActionDetails details = detailsBuilder.build();
370 | assert (details.getExecuteResponse() != null);
371 | assert (details.isFailed());
372 | };
373 |
374 | @Test
375 | public void FailedActionsEmpty() throws IOException, ParamException {
376 | ActionGrouping.Builder grouping = new ActionGrouping.Builder();
377 | List result = grouping.build().failedActions();
378 | assert (result.isEmpty());
379 | }
380 |
381 | @Test
382 | public void FailedActionsAllPass() throws IOException, ParamException {
383 | ActionGrouping.Builder grouping = new ActionGrouping.Builder();
384 | grouping.addLogEntry(getLogEntry("actionId", "Execute", 10, makeWatch(actionResultSuccess)));
385 | List result = grouping.build().failedActions();
386 | assert (result.isEmpty());
387 | }
388 |
389 | @Test
390 | public void FailedActionsOneFail() throws IOException, ParamException {
391 | ActionGrouping.Builder grouping = new ActionGrouping.Builder();
392 | grouping.addLogEntry(
393 | addDigest(
394 | getLogEntry("12345", "Execute", 10, makeExecute(actionResultSuccess)), "12345/56"));
395 | grouping.addLogEntry(
396 | addDigest(getLogEntry("987", "Execute", 10, makeExecute(actionResultFail)), "987/22"));
397 | grouping.addLogEntry(
398 | addDigest(getLogEntry("345", "Execute", 10, makeExecute(actionResultSuccess)), "345/1"));
399 | List result = grouping.build().failedActions();
400 |
401 | List expected = Arrays.asList(toDigest("987/22"));
402 | assertThat(result).isEqualTo(expected);
403 | }
404 |
405 | @Test
406 | public void FailedActionsManyFail() throws IOException, ParamException {
407 | ActionGrouping.Builder grouping = new ActionGrouping.Builder();
408 | grouping.addLogEntry(
409 | addDigest(getLogEntry("12345", "Execute", 10, makeExecute(actionResultFail)), "12345/56"));
410 | grouping.addLogEntry(
411 | addDigest(getLogEntry("987", "Execute", 10, makeExecute(actionResultFail)), "987/22"));
412 | grouping.addLogEntry(
413 | addDigest(getLogEntry("345", "Execute", 10, makeExecute(actionResultFail)), "345/1"));
414 | List result = grouping.build().failedActions();
415 |
416 | List expected =
417 | Arrays.asList(toDigest("12345/56"), toDigest("987/22"), toDigest("345/1"));
418 | assertThat(result).isEqualTo(expected);
419 | }
420 |
421 | @Test
422 | public void FailedActionsDifferentResults() throws IOException, ParamException {
423 | ActionGrouping.Builder grouping = new ActionGrouping.Builder();
424 | grouping.addLogEntry(
425 | addDigest(
426 | getLogEntry("12345", "Execute", 10, makeGetActionResult(actionResultFail)),
427 | "12345/56"));
428 | grouping.addLogEntry(
429 | addDigest(
430 | getLogEntry("987", "Execute", 10, makeGetActionResult(actionResultSuccess)), "987/22"));
431 | grouping.addLogEntry(
432 | addDigest(getLogEntry("345", "Execute", 10, makeExecute(actionResultFail)), "345/1"));
433 | List result = grouping.build().failedActions();
434 |
435 | List expected = Arrays.asList(toDigest("12345/56"), toDigest("345/1"));
436 | assertThat(result).isEqualTo(expected);
437 | }
438 |
439 | @Test
440 | public void FailedActionsGetsDigestFromGetActionResult() throws IOException, ParamException {
441 | ActionGrouping.Builder grouping = new ActionGrouping.Builder();
442 | grouping.addLogEntry(makeFailedGetActionResult(10, "12345/88"));
443 | grouping.addLogEntry(getLogEntry("12345", "Execute", 10, makeWatch(actionResultFail)));
444 | List result = grouping.build().failedActions();
445 |
446 | List expected = Arrays.asList(toDigest("12345/88"));
447 | assertThat(result).isEqualTo(expected);
448 | }
449 |
450 | @Test
451 | public void FailedActionsGetsDigestFromExecute() throws IOException, ParamException {
452 | ActionGrouping.Builder grouping = new ActionGrouping.Builder();
453 | grouping.addLogEntry(
454 | addDigest(
455 | getLogEntry("12345", "Execute", 10, makeExecute(ActionResult.getDefaultInstance())),
456 | "12345/1"));
457 | grouping.addLogEntry(getLogEntry("12345", "Execute", 10, makeWatch(actionResultFail)));
458 | List result = grouping.build().failedActions();
459 |
460 | List expected = Arrays.asList(toDigest("12345/1"));
461 | assertThat(result).isEqualTo(expected);
462 | }
463 |
464 | @Test
465 | public void FailedActionsDoesntGetDigestFromWrongExecute() throws IOException, ParamException {
466 | ActionGrouping.Builder grouping = new ActionGrouping.Builder();
467 | grouping.addLogEntry(
468 | addDigest(
469 | getLogEntry(
470 | "wrong_action", "Execute", 10, makeExecute(ActionResult.getDefaultInstance())),
471 | "12345/1"));
472 | grouping.addLogEntry(getLogEntry("12345", "Execute", 10, makeWatch(actionResultFail)));
473 | List result = grouping.build().failedActions();
474 |
475 | assertThat(result).isEmpty();
476 | }
477 |
478 | @FunctionalInterface
479 | public interface ThrowingRunnable{
480 | void run() throws Exception;
481 | }
482 |
483 |
484 |
485 | private String captureStderr(ThrowingRunnable r) throws Exception {
486 | PrintStream oldErr = System.err;
487 |
488 | String result = null;
489 |
490 | final ByteArrayOutputStream stream = new ByteArrayOutputStream();
491 | try (PrintStream ps = new PrintStream(stream)) {
492 | System.setErr(ps);
493 | r.run();
494 | ps.flush();
495 | result = stream.toString();
496 | } finally {
497 | System.setErr(oldErr);
498 | }
499 | return result;
500 | }
501 |
502 | @Test
503 | public void DigestMismatchError() throws Exception {
504 | // Mismatching action digest causes an error
505 | String response = captureStderr(() -> {
506 | ActionGrouping.Builder grouping = new ActionGrouping.Builder();
507 | grouping.addLogEntry(
508 | addDigest(
509 | getLogEntry(
510 | "12345", "Execute", 10, makeExecute(ActionResult.getDefaultInstance())),
511 | "12345/1"));
512 | grouping.build();
513 | });
514 |
515 | assertThat(response).isEmpty();
516 | }
517 |
518 | @Test
519 | public void DigestOk() throws Exception {
520 | // Matching action digest is ok
521 | String response = captureStderr(() -> {
522 | ActionGrouping.Builder grouping = new ActionGrouping.Builder();
523 | grouping.addLogEntry(
524 | addDigest(
525 | getLogEntry(
526 | "12345", "Execute", 10, makeExecute(ActionResult.getDefaultInstance())),
527 | "wrong/1"));
528 | grouping.build();
529 | });
530 |
531 | assertThat(response).contains("Warning: bad digest:");
532 | assertThat(response).contains("wrong");
533 | assertThat(response).contains("doesn't match action Id 12345");
534 | }
535 |
536 | @Test
537 | public void MultipleActionDigestOk() throws Exception {
538 | // Mismatching action digests across actions cause error
539 | String response = captureStderr(() -> {
540 | ActionGrouping.Builder grouping = new ActionGrouping.Builder();
541 | grouping.addLogEntry(makeFailedGetActionResult(10, "12345/1"));
542 | grouping.addLogEntry(
543 | addDigest(
544 | getLogEntry(
545 | "12345", "Execute", 10, makeExecute(ActionResult.getDefaultInstance())),
546 | "12345/1"));
547 | grouping.build();
548 | });
549 |
550 | assertThat(response).isEmpty();
551 | }
552 |
553 | @Test
554 | public void MultipleActionDigestMismatch() throws Exception {
555 | // Matching action digests across actions are ok
556 | String response = captureStderr(() -> {
557 | ActionGrouping.Builder grouping = new ActionGrouping.Builder();
558 | grouping.addLogEntry(makeFailedGetActionResult(10, "12345/1"));
559 | grouping.addLogEntry(
560 | addDigest(
561 | getLogEntry(
562 | "12345", "Execute", 10, makeExecute(ActionResult.getDefaultInstance())),
563 | "12345/2"));
564 | grouping.build();
565 | });
566 |
567 | assertThat(response).contains("conflicting digests");
568 | assertThat(response).contains("12345");
569 | assertThat(response).contains("size_bytes: 2");
570 | assertThat(response).contains("size_bytes: 1");
571 | }
572 | }
573 |
--------------------------------------------------------------------------------
/src/test/java/com/google/devtools/build/remote/client/BUILD:
--------------------------------------------------------------------------------
1 | java_test(
2 | name = "GrpcRemoteCacheTest",
3 | srcs = [
4 | "FakeImmutableCacheByteStreamImpl.java",
5 | "GrpcRemoteCacheTest.java",
6 | ],
7 | deps = [
8 | "//src/main/java/com/google/devtools/build/remote/client",
9 | "//third_party/remote-apis:build_bazel_remote_execution_v2_remote_execution_java_grpc",
10 | "@googleapis//google/bytestream:bytestream_java_grpc",
11 | "@googleapis//google/bytestream:bytestream_java_proto",
12 | "@maven//:com_google_guava_guava",
13 | "@maven//:com_google_http_client_google_http_client",
14 | "@maven//:com_google_http_client_google_http_client_jackson2",
15 | "@maven//:com_google_jimfs_jimfs",
16 | "@maven//:com_google_protobuf_protobuf_java",
17 | "@maven//:com_google_truth_truth",
18 | "@maven//:io_grpc_grpc_api",
19 | "@maven//:io_grpc_grpc_context",
20 | "@maven//:io_grpc_grpc_core",
21 | "@maven//:io_grpc_grpc_inprocess",
22 | "@maven//:io_grpc_grpc_stub",
23 | "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_java_proto",
24 | ],
25 | )
26 |
27 | java_test(
28 | name = "ShellEscaperTest",
29 | srcs = ["ShellEscaperTest.java"],
30 | deps = [
31 | "@maven//:com_google_truth_truth",
32 | "//src/main/java/com/google/devtools/build/remote/client",
33 | ],
34 | )
35 |
36 | java_test(
37 | name = "ActionGroupingTest",
38 | srcs = ["ActionGroupingTest.java"],
39 | deps = [
40 | "//src/main/java/com/google/devtools/build/remote/client",
41 | "//src/main/proto:remote_execution_log_java_proto",
42 | "@googleapis//google/longrunning:longrunning_java_proto",
43 | "@maven//:com_google_protobuf_protobuf_java",
44 | "@maven//:com_google_protobuf_protobuf_java_util",
45 | "@maven//:com_google_truth_truth",
46 | "@maven//:io_grpc_grpc_api",
47 | "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_java_proto",
48 | ],
49 | )
50 |
51 | java_test(
52 | name = "DockerUtilTest",
53 | srcs = ["DockerUtilTest.java"],
54 | deps = [
55 | "//src/main/java/com/google/devtools/build/remote/client",
56 | "@maven//:com_google_guava_guava",
57 | "@maven//:com_google_truth_truth",
58 | "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_java_proto",
59 | ],
60 | )
61 |
--------------------------------------------------------------------------------
/src/test/java/com/google/devtools/build/remote/client/DockerUtilTest.java:
--------------------------------------------------------------------------------
1 | // Copyright 2018 The Bazel Authors. All rights reserved.
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | package com.google.devtools.build.remote.client;
15 |
16 | import static com.google.common.truth.Truth.assertThat;
17 |
18 | import build.bazel.remote.execution.v2.Command;
19 | import build.bazel.remote.execution.v2.Command.EnvironmentVariable;
20 | import build.bazel.remote.execution.v2.Platform;
21 | import build.bazel.remote.execution.v2.Platform.Property;
22 | import com.google.common.hash.Hashing;
23 | import org.junit.Test;
24 | import org.junit.runner.RunWith;
25 | import org.junit.runners.JUnit4;
26 |
27 | /** Tests for {@link DockerUtil}. */
28 | @RunWith(JUnit4.class)
29 | public class DockerUtilTest {
30 | private static final DigestUtil DIGEST_UTIL = new DigestUtil(Hashing.sha256());
31 |
32 | private static class MockUidGetter extends DockerUtil.UidGetter {
33 | long uid;
34 |
35 | public MockUidGetter (long uid) {
36 | this.uid = uid;
37 | }
38 |
39 | @Override
40 | public long getUid() {
41 | return uid;
42 | }
43 | }
44 |
45 | static final Command command =
46 | Command.newBuilder()
47 | .addArguments("/bin/echo")
48 | .addArguments("hello")
49 | .addArguments("escape<'>")
50 | .addEnvironmentVariables(
51 | EnvironmentVariable.newBuilder().setName("PATH").setValue("/home/test"))
52 | .setPlatform(
53 | Platform.newBuilder()
54 | .addProperties(
55 | Property.newBuilder()
56 | .setName("container-image")
57 | .setValue("docker://gcr.io/image")))
58 | .build();
59 |
60 | @Test
61 | public void testGetDockerCommandNoUid() {
62 | DockerUtil util = new DockerUtil(new MockUidGetter(-1));
63 | String commandLine = util.getDockerCommand(command, "/tmp/test");
64 |
65 | assertThat(commandLine)
66 | .isEqualTo(
67 | "docker run -v /tmp/test:/tmp/test-docker -w /tmp/test-docker -e 'PATH=/home/test' "
68 | + "gcr.io/image /bin/echo hello 'escape<'\\''>'");
69 | }
70 |
71 | @Test
72 | public void testGetDockerCommandUid() {
73 | DockerUtil util = new DockerUtil(new MockUidGetter(14242));
74 | String commandLine = util.getDockerCommand(command, "/tmp/test");
75 |
76 | if (System.getProperty("os.name").startsWith("Windows")) {
77 | assertThat(commandLine)
78 | .isEqualTo(
79 | "docker run -v /tmp/test:/tmp/test-docker "
80 | + "-w /tmp/test-docker -e 'PATH=/home/test' "
81 | + "gcr.io/image /bin/echo hello 'escape<'\\''>'");
82 | } else {
83 | assertThat(commandLine)
84 | .isEqualTo(
85 | "docker run -u 14242 -v /tmp/test:/tmp/test-docker "
86 | + "-w /tmp/test-docker -e 'PATH=/home/test' "
87 | + "gcr.io/image /bin/echo hello 'escape<'\\''>'");
88 | }
89 | }
90 |
91 | @Test(expected = IllegalArgumentException.class)
92 | public void testGetDockerCommandNoPlatformFail() {
93 | Command command =
94 | Command.newBuilder()
95 | .addArguments("/bin/echo")
96 | .addArguments("hello")
97 | .addArguments("escape<'>")
98 | .addEnvironmentVariables(
99 | EnvironmentVariable.newBuilder().setName("PATH").setValue("/home/test"))
100 | .build();
101 | DockerUtil util = new DockerUtil(new MockUidGetter(-1));
102 | util.getDockerCommand(command, "/tmp/test");
103 | }
104 | }
105 |
--------------------------------------------------------------------------------
/src/test/java/com/google/devtools/build/remote/client/FakeImmutableCacheByteStreamImpl.java:
--------------------------------------------------------------------------------
1 | // Copyright 2017 The Bazel Authors. All rights reserved.
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | package com.google.devtools.build.remote.client;
15 |
16 | import static com.google.common.truth.Truth.assertThat;
17 |
18 | import build.bazel.remote.execution.v2.Digest;
19 | import com.google.bytestream.ByteStreamGrpc.ByteStreamImplBase;
20 | import com.google.bytestream.ByteStreamProto.ReadRequest;
21 | import com.google.bytestream.ByteStreamProto.ReadResponse;
22 | import com.google.common.collect.ImmutableMap;
23 | import com.google.protobuf.ByteString;
24 | import io.grpc.stub.StreamObserver;
25 | import java.util.HashMap;
26 | import java.util.Map;
27 |
28 | class FakeImmutableCacheByteStreamImpl extends ByteStreamImplBase {
29 | private final Map cannedReplies;
30 | private final Map numErrors;
31 | // Start returning the correct response after this number of errors is reached.
32 | private static final int MAX_ERRORS = 3;
33 |
34 | public FakeImmutableCacheByteStreamImpl(Map contents) {
35 | ImmutableMap.Builder b = ImmutableMap.builder();
36 | for (Map.Entry e : contents.entrySet()) {
37 | Object obj = e.getValue();
38 | ByteString data;
39 | if (obj instanceof String) {
40 | data = ByteString.copyFromUtf8((String) obj);
41 | } else if (obj instanceof ByteString) {
42 | data = (ByteString) obj;
43 | } else {
44 | throw new AssertionError(
45 | "expected object to be either a String or a ByteString, got a "
46 | + obj.getClass().getCanonicalName());
47 | }
48 | b.put(
49 | ReadRequest.newBuilder()
50 | .setResourceName("blobs/" + e.getKey().getHash() + "/" + e.getKey().getSizeBytes())
51 | .build(),
52 | ReadResponse.newBuilder().setData(data).build());
53 | }
54 | cannedReplies = b.build();
55 | numErrors = new HashMap<>();
56 | }
57 |
58 | @Override
59 | public void read(ReadRequest request, StreamObserver responseObserver) {
60 | assertThat(cannedReplies.keySet()).contains(request);
61 | responseObserver.onNext(cannedReplies.get(request));
62 | responseObserver.onCompleted();
63 | }
64 | }
65 |
--------------------------------------------------------------------------------
/src/test/java/com/google/devtools/build/remote/client/GrpcRemoteCacheTest.java:
--------------------------------------------------------------------------------
1 | // Copyright 2018 The Bazel Authors. All rights reserved.
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package com.google.devtools.build.remote.client;
16 |
17 | import static com.google.common.truth.Truth.assertThat;
18 | import static java.nio.charset.StandardCharsets.UTF_8;
19 |
20 | import build.bazel.remote.execution.v2.ContentAddressableStorageGrpc.ContentAddressableStorageImplBase;
21 | import build.bazel.remote.execution.v2.Digest;
22 | import build.bazel.remote.execution.v2.Directory;
23 | import build.bazel.remote.execution.v2.DirectoryNode;
24 | import build.bazel.remote.execution.v2.FileNode;
25 | import build.bazel.remote.execution.v2.GetTreeRequest;
26 | import build.bazel.remote.execution.v2.GetTreeResponse;
27 | import build.bazel.remote.execution.v2.OutputDirectory;
28 | import build.bazel.remote.execution.v2.RequestMetadata;
29 | import build.bazel.remote.execution.v2.ToolDetails;
30 | import build.bazel.remote.execution.v2.Tree;
31 | import com.google.api.client.json.GenericJson;
32 | import com.google.api.client.json.jackson2.JacksonFactory;
33 | import com.google.bytestream.ByteStreamGrpc.ByteStreamImplBase;
34 | import com.google.bytestream.ByteStreamProto.ReadRequest;
35 | import com.google.bytestream.ByteStreamProto.ReadResponse;
36 | import com.google.common.collect.ImmutableList;
37 | import com.google.common.collect.ImmutableMap;
38 | import com.google.common.hash.Hashing;
39 | import com.google.common.jimfs.Configuration;
40 | import com.google.common.jimfs.Jimfs;
41 | import com.google.protobuf.ByteString;
42 | import io.grpc.CallCredentials;
43 | import io.grpc.CallOptions;
44 | import io.grpc.Channel;
45 | import io.grpc.ClientCall;
46 | import io.grpc.ClientInterceptor;
47 | import io.grpc.ClientInterceptors;
48 | import io.grpc.Context;
49 | import io.grpc.MethodDescriptor;
50 | import io.grpc.Server;
51 | import io.grpc.inprocess.InProcessChannelBuilder;
52 | import io.grpc.inprocess.InProcessServerBuilder;
53 | import io.grpc.stub.StreamObserver;
54 | import io.grpc.util.MutableHandlerRegistry;
55 | import java.io.IOException;
56 | import java.nio.file.FileSystem;
57 | import java.nio.file.Files;
58 | import java.nio.file.Path;
59 | import java.nio.file.attribute.PosixFilePermission;
60 | import org.junit.After;
61 | import org.junit.Before;
62 | import org.junit.Test;
63 | import org.junit.runner.RunWith;
64 | import org.junit.runners.JUnit4;
65 |
66 | /** Tests for {@link GrpcRemoteCache}. */
67 | @RunWith(JUnit4.class)
68 | public class GrpcRemoteCacheTest {
69 |
70 | private static final DigestUtil DIGEST_UTIL = new DigestUtil(Hashing.sha256());
71 |
72 | private final String fakeServerName = "fake server for " + getClass();
73 | private final MutableHandlerRegistry serviceRegistry = new MutableHandlerRegistry();
74 | private final FileSystem fs =
75 | Jimfs.newFileSystem(Configuration.unix().toBuilder().setAttributeViews("posix").build());
76 | private Path execRoot;
77 | private Server fakeServer;
78 | private Context withEmptyMetadata;
79 | private Context prevCtx;
80 |
81 | @Before
82 | public final void setUp() throws Exception {
83 | // Use a mutable service registry for later registering the service impl for each test case.
84 | fakeServer =
85 | InProcessServerBuilder.forName(fakeServerName)
86 | .fallbackHandlerRegistry(serviceRegistry)
87 | .directExecutor()
88 | .build()
89 | .start();
90 |
91 | execRoot = fs.getPath("/exec/root/");
92 | RequestMetadata testMetadata =
93 | RequestMetadata.newBuilder()
94 | .setToolDetails(ToolDetails.newBuilder().setToolName("TEST"))
95 | .build();
96 | withEmptyMetadata = TracingMetadataUtils.contextWithMetadata(testMetadata);
97 | prevCtx = withEmptyMetadata.attach();
98 | }
99 |
100 | @After
101 | public void tearDown() throws Exception {
102 | withEmptyMetadata.detach(prevCtx);
103 | fakeServer.shutdownNow();
104 | fakeServer.awaitTermination();
105 | }
106 |
107 | private static class CallCredentialsInterceptor implements ClientInterceptor {
108 | private final CallCredentials credentials;
109 |
110 | public CallCredentialsInterceptor(CallCredentials credentials) {
111 | this.credentials = credentials;
112 | }
113 |
114 | @Override
115 | public ClientCall interceptCall(
116 | MethodDescriptor method, CallOptions callOptions, Channel next) {
117 | assertThat(callOptions.getCredentials()).isEqualTo(credentials);
118 | // Remove the call credentials to allow testing with dummy ones.
119 | return next.newCall(method, callOptions.withCallCredentials(null));
120 | }
121 | }
122 |
123 | private GrpcRemoteCache newClient() throws IOException {
124 | AuthAndTLSOptions authTlsOptions = new AuthAndTLSOptions();
125 | authTlsOptions.useGoogleDefaultCredentials = true;
126 | authTlsOptions.googleCredentials = "/exec/root/creds.json";
127 | authTlsOptions.googleAuthScopes = ImmutableList.of("dummy.scope");
128 |
129 | GenericJson json = new GenericJson();
130 | json.put("type", "authorized_user");
131 | json.put("client_id", "some_client");
132 | json.put("client_secret", "foo");
133 | json.put("refresh_token", "bar");
134 | FileSystem scratchFs = Jimfs.newFileSystem(Configuration.unix());
135 | Path credsPath = scratchFs.getPath(authTlsOptions.googleCredentials);
136 | Files.createDirectories(credsPath.getParent());
137 | Files.write(credsPath, new JacksonFactory().toString(json).getBytes());
138 |
139 | CallCredentials creds =
140 | GoogleAuthUtils.newCallCredentials(
141 | Files.newInputStream(credsPath), authTlsOptions.googleAuthScopes);
142 |
143 | RemoteOptions remoteOptions = new RemoteOptions();
144 | return new GrpcRemoteCache(
145 | ClientInterceptors.intercept(
146 | InProcessChannelBuilder.forName(fakeServerName).directExecutor().build(),
147 | ImmutableList.of(new CallCredentialsInterceptor(creds))),
148 | creds,
149 | remoteOptions,
150 | DIGEST_UTIL);
151 | }
152 |
153 | // Returns whether a path/file is executable or not.
154 | private boolean isExecutable(Path path) throws IOException {
155 | return Files.getPosixFilePermissions(path).contains(PosixFilePermission.OWNER_EXECUTE);
156 | }
157 |
158 | @Test
159 | public void testDownloadEmptyBlob() throws Exception {
160 | GrpcRemoteCache client = newClient();
161 | Digest emptyDigest = DIGEST_UTIL.compute(new byte[0]);
162 | // Will not call the mock Bytestream interface at all.
163 | assertThat(client.downloadBlob(emptyDigest)).isEmpty();
164 | }
165 |
166 | @Test
167 | public void testDownloadBlobSingleChunk() throws Exception {
168 | final GrpcRemoteCache client = newClient();
169 | final Digest digest = DIGEST_UTIL.computeAsUtf8("abcdefg");
170 | serviceRegistry.addService(
171 | new ByteStreamImplBase() {
172 | @Override
173 | public void read(ReadRequest request, StreamObserver responseObserver) {
174 | assertThat(request.getResourceName().contains(digest.getHash())).isTrue();
175 | responseObserver.onNext(
176 | ReadResponse.newBuilder().setData(ByteString.copyFromUtf8("abcdefg")).build());
177 | responseObserver.onCompleted();
178 | }
179 | });
180 | assertThat(new String(client.downloadBlob(digest), UTF_8)).isEqualTo("abcdefg");
181 | }
182 |
183 | @Test
184 | public void testDownloadBlobMultipleChunks() throws Exception {
185 | final GrpcRemoteCache client = newClient();
186 | final Digest digest = DIGEST_UTIL.computeAsUtf8("abcdefg");
187 | serviceRegistry.addService(
188 | new ByteStreamImplBase() {
189 | @Override
190 | public void read(ReadRequest request, StreamObserver responseObserver) {
191 | assertThat(request.getResourceName().contains(digest.getHash())).isTrue();
192 | responseObserver.onNext(
193 | ReadResponse.newBuilder().setData(ByteString.copyFromUtf8("abc")).build());
194 | responseObserver.onNext(
195 | ReadResponse.newBuilder().setData(ByteString.copyFromUtf8("def")).build());
196 | responseObserver.onNext(
197 | ReadResponse.newBuilder().setData(ByteString.copyFromUtf8("g")).build());
198 | responseObserver.onCompleted();
199 | }
200 | });
201 | assertThat(new String(client.downloadBlob(digest), UTF_8)).isEqualTo("abcdefg");
202 | }
203 |
204 | @Test
205 | public void testDownloadDirectoryEmpty() throws Exception {
206 | GrpcRemoteCache client = newClient();
207 |
208 | Directory dirMessage = Directory.getDefaultInstance();
209 | Digest dirDigest = DIGEST_UTIL.compute(dirMessage);
210 | serviceRegistry.addService(
211 | new FakeImmutableCacheByteStreamImpl(
212 | ImmutableMap.of(dirDigest, dirMessage.toByteString())));
213 | serviceRegistry.addService(
214 | new ContentAddressableStorageImplBase() {
215 | @Override
216 | public void getTree(
217 | GetTreeRequest request, StreamObserver responseObserver) {
218 | assertThat(request.getRootDigest()).isEqualTo(dirDigest);
219 | responseObserver.onNext(
220 | GetTreeResponse.newBuilder().addDirectories(dirMessage).build());
221 | responseObserver.onCompleted();
222 | }
223 | });
224 | client.downloadDirectory(execRoot.resolve("test"), dirDigest);
225 | assertThat(Files.exists(execRoot.resolve("test"))).isTrue();
226 | }
227 |
228 | @Test
229 | public void testDownloadDirectory() throws Exception {
230 | GrpcRemoteCache client = newClient();
231 | Digest fooDigest = DIGEST_UTIL.computeAsUtf8("foo-contents");
232 |
233 | Directory barMessage =
234 | Directory.newBuilder()
235 | .addFiles(
236 | FileNode.newBuilder().setDigest(fooDigest).setName("foo").setDigest(fooDigest))
237 | .build();
238 | Digest barDigest = DIGEST_UTIL.compute(barMessage);
239 |
240 | Directory dirMessage =
241 | Directory.newBuilder()
242 | .addDirectories(DirectoryNode.newBuilder().setDigest(barDigest).setName("bar"))
243 | .addFiles(
244 | FileNode.newBuilder().setDigest(fooDigest).setName("foo").setIsExecutable(true))
245 | .build();
246 | Digest dirDigest = DIGEST_UTIL.compute(dirMessage);
247 | serviceRegistry.addService(
248 | new FakeImmutableCacheByteStreamImpl(
249 | ImmutableMap.of(dirDigest, dirMessage.toByteString(), fooDigest, "foo-contents")));
250 | serviceRegistry.addService(
251 | new ContentAddressableStorageImplBase() {
252 | @Override
253 | public void getTree(
254 | GetTreeRequest request, StreamObserver responseObserver) {
255 | assertThat(request.getRootDigest()).isEqualTo(dirDigest);
256 | responseObserver.onNext(
257 | GetTreeResponse.newBuilder()
258 | .addDirectories(dirMessage)
259 | .addDirectories(barMessage)
260 | .build());
261 | responseObserver.onCompleted();
262 | }
263 | });
264 |
265 | client.downloadDirectory(execRoot.resolve("test"), dirDigest);
266 | assertThat(Files.exists(execRoot.resolve("test"))).isTrue();
267 | assertThat(Files.exists(execRoot.resolve("test/foo"))).isTrue();
268 | assertThat(Files.exists(execRoot.resolve("test/bar"))).isTrue();
269 | assertThat(Files.exists(execRoot.resolve("test/bar/foo"))).isTrue();
270 | assertThat(Files.isRegularFile(execRoot.resolve("test/foo"))).isTrue();
271 | assertThat(Files.isDirectory(execRoot.resolve("test/bar"))).isTrue();
272 | assertThat(Files.isRegularFile(execRoot.resolve("test/bar/foo"))).isTrue();
273 | if (!System.getProperty("os.name").startsWith("Windows")) {
274 | assertThat(isExecutable(execRoot.resolve("test/foo"))).isTrue();
275 | assertThat(isExecutable(execRoot.resolve("test/bar/foo"))).isFalse();
276 | }
277 | }
278 |
279 | @Test
280 | public void testGetTree() throws Exception {
281 | GrpcRemoteCache client = newClient();
282 | Directory quxMessage = Directory.getDefaultInstance();
283 | Directory barMessage =
284 | Directory.newBuilder().addFiles(FileNode.newBuilder().setName("test")).build();
285 | Digest quxDigest = DIGEST_UTIL.compute(quxMessage);
286 | Digest barDigest = DIGEST_UTIL.compute(barMessage);
287 | Directory fooMessage =
288 | Directory.newBuilder()
289 | .addDirectories(DirectoryNode.newBuilder().setDigest(quxDigest).setName("qux"))
290 | .addDirectories(DirectoryNode.newBuilder().setDigest(barDigest).setName("bar"))
291 | .build();
292 | Digest fooDigest = DIGEST_UTIL.compute(fooMessage);
293 | serviceRegistry.addService(
294 | new FakeImmutableCacheByteStreamImpl(
295 | ImmutableMap.of(fooDigest, fooMessage.toByteString())));
296 | GetTreeResponse response1 =
297 | GetTreeResponse.newBuilder().addDirectories(quxMessage).setNextPageToken("token").build();
298 | GetTreeResponse response2 = GetTreeResponse.newBuilder().addDirectories(barMessage).build();
299 | serviceRegistry.addService(
300 | new ContentAddressableStorageImplBase() {
301 | @Override
302 | public void getTree(
303 | GetTreeRequest request, StreamObserver responseObserver) {
304 | responseObserver.onNext(response1);
305 | responseObserver.onNext(response2);
306 | responseObserver.onCompleted();
307 | }
308 | });
309 | Tree tree = client.getTree(fooDigest);
310 | assertThat(tree.getRoot()).isEqualTo(fooMessage);
311 | assertThat(tree.getChildrenList()).containsExactly(quxMessage, barMessage);
312 | }
313 |
314 | @Test
315 | public void testDownloadOutputDirectory() throws Exception {
316 | GrpcRemoteCache client = newClient();
317 | Digest fooDigest = DIGEST_UTIL.computeAsUtf8("foo-contents");
318 | Digest quxDigest = DIGEST_UTIL.computeAsUtf8("qux-contents");
319 | Tree barTreeMessage =
320 | Tree.newBuilder()
321 | .setRoot(
322 | Directory.newBuilder()
323 | .addFiles(
324 | FileNode.newBuilder()
325 | .setName("qux")
326 | .setDigest(quxDigest)
327 | .setIsExecutable(true)))
328 | .build();
329 | Digest barTreeDigest = DIGEST_UTIL.compute(barTreeMessage);
330 | OutputDirectory barDirMessage =
331 | OutputDirectory.newBuilder().setPath("test/bar").setTreeDigest(barTreeDigest).build();
332 | Digest barDirDigest = DIGEST_UTIL.compute(barDirMessage);
333 | serviceRegistry.addService(
334 | new FakeImmutableCacheByteStreamImpl(
335 | ImmutableMap.of(
336 | fooDigest,
337 | "foo-contents",
338 | barTreeDigest,
339 | barTreeMessage.toByteString(),
340 | quxDigest,
341 | "qux-contents",
342 | barDirDigest,
343 | barDirMessage.toByteString())));
344 |
345 | client.downloadOutputDirectory(barDirMessage, execRoot.resolve("test/bar"));
346 |
347 | assertThat(Files.exists(execRoot.resolve("test/bar"))).isTrue();
348 | assertThat(Files.isDirectory(execRoot.resolve("test/bar"))).isTrue();
349 | assertThat(Files.exists(execRoot.resolve("test/bar/qux"))).isTrue();
350 | assertThat(Files.isRegularFile(execRoot.resolve("test/bar/qux"))).isTrue();
351 | if (!System.getProperty("os.name").startsWith("Windows")) {
352 | assertThat(isExecutable(execRoot.resolve("test/bar/qux"))).isTrue();
353 | }
354 | }
355 |
356 | @Test
357 | public void testDownloadOutputDirectoryEmpty() throws Exception {
358 | GrpcRemoteCache client = newClient();
359 |
360 | Tree barTreeMessage = Tree.newBuilder().setRoot(Directory.newBuilder()).build();
361 | Digest barTreeDigest = DIGEST_UTIL.compute(barTreeMessage);
362 | OutputDirectory barDirMessage =
363 | OutputDirectory.newBuilder().setPath("test/bar").setTreeDigest(barTreeDigest).build();
364 | Digest barDirDigest = DIGEST_UTIL.compute(barDirMessage);
365 | serviceRegistry.addService(
366 | new FakeImmutableCacheByteStreamImpl(
367 | ImmutableMap.of(
368 | barTreeDigest, barTreeMessage.toByteString(),
369 | barDirDigest, barDirMessage.toByteString())));
370 |
371 | client.downloadOutputDirectory(barDirMessage, execRoot.resolve("test/bar"));
372 |
373 | assertThat(Files.exists(execRoot.resolve("test/bar"))).isTrue();
374 | assertThat(Files.isDirectory(execRoot.resolve("test/bar"))).isTrue();
375 | }
376 |
377 | @Test
378 | public void testDownloadOutputDirectoryNested() throws Exception {
379 | GrpcRemoteCache client = newClient();
380 | Digest fooDigest = DIGEST_UTIL.computeAsUtf8("foo-contents");
381 | Digest quxDigest = DIGEST_UTIL.computeAsUtf8("qux-contents");
382 | Directory wobbleDirMessage =
383 | Directory.newBuilder()
384 | .addFiles(FileNode.newBuilder().setName("qux").setDigest(quxDigest))
385 | .build();
386 | Digest wobbleDigest = DIGEST_UTIL.compute(wobbleDirMessage);
387 | Tree barTreeMessage =
388 | Tree.newBuilder()
389 | .setRoot(
390 | Directory.newBuilder()
391 | .addFiles(FileNode.newBuilder().setName("qux").setDigest(quxDigest))
392 | .addDirectories(
393 | DirectoryNode.newBuilder().setName("wobble").setDigest(wobbleDigest)))
394 | .addChildren(wobbleDirMessage)
395 | .build();
396 | Digest barTreeDigest = DIGEST_UTIL.compute(barTreeMessage);
397 | OutputDirectory barDirMessage =
398 | OutputDirectory.newBuilder().setPath("test/bar").setTreeDigest(barTreeDigest).build();
399 | Digest barDirDigest = DIGEST_UTIL.compute(barDirMessage);
400 | serviceRegistry.addService(
401 | new FakeImmutableCacheByteStreamImpl(
402 | ImmutableMap.of(
403 | fooDigest,
404 | "foo-contents",
405 | barTreeDigest,
406 | barTreeMessage.toByteString(),
407 | quxDigest,
408 | "qux-contents",
409 | barDirDigest,
410 | barDirMessage.toByteString())));
411 |
412 | client.downloadOutputDirectory(barDirMessage, execRoot.resolve("test/bar"));
413 |
414 | assertThat(Files.exists(execRoot.resolve("test/bar"))).isTrue();
415 | assertThat(Files.isDirectory(execRoot.resolve("test/bar"))).isTrue();
416 |
417 | assertThat(Files.exists(execRoot.resolve("test/bar/wobble"))).isTrue();
418 | assertThat(Files.isDirectory(execRoot.resolve("test/bar/wobble"))).isTrue();
419 |
420 | assertThat(Files.exists(execRoot.resolve("test/bar/wobble/qux"))).isTrue();
421 | assertThat(Files.isRegularFile(execRoot.resolve("test/bar/wobble/qux"))).isTrue();
422 |
423 | assertThat(Files.exists(execRoot.resolve("test/bar/qux"))).isTrue();
424 | assertThat(Files.isRegularFile(execRoot.resolve("test/bar/qux"))).isTrue();
425 | if (!System.getProperty("os.name").startsWith("Windows")) {
426 | assertThat(isExecutable(execRoot.resolve("test/bar/wobble/qux"))).isFalse();
427 | assertThat(isExecutable(execRoot.resolve("test/bar/qux"))).isFalse();
428 | }
429 | }
430 | }
431 |
--------------------------------------------------------------------------------
/src/test/java/com/google/devtools/build/remote/client/ShellEscaperTest.java:
--------------------------------------------------------------------------------
1 | // Copyright 2018 The Bazel Authors. All rights reserved.
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | package com.google.devtools.build.remote.client;
15 |
16 | import static com.google.common.truth.Truth.assertThat;
17 | import static com.google.devtools.build.remote.client.ShellEscaper.escapeString;
18 |
19 | import java.util.Arrays;
20 | import org.junit.Test;
21 | import org.junit.runner.RunWith;
22 | import org.junit.runners.JUnit4;
23 |
24 | /** Tests for {@link ShellEscaper}. */
25 | @RunWith(JUnit4.class)
26 | public class ShellEscaperTest {
27 |
28 | @Test
29 | public void shellEscape() throws Exception {
30 | assertThat(escapeString("")).isEqualTo("''");
31 | assertThat(escapeString("foo")).isEqualTo("foo");
32 | assertThat(escapeString("foo bar")).isEqualTo("'foo bar'");
33 | assertThat(escapeString("'foo'")).isEqualTo("''\\''foo'\\'''");
34 | assertThat(escapeString("\\'foo\\'")).isEqualTo("'\\'\\''foo\\'\\'''");
35 | assertThat(escapeString("${filename%.c}.o")).isEqualTo("'${filename%.c}.o'");
36 | assertThat(escapeString("")).isEqualTo("''");
37 | }
38 |
39 | @Test
40 | public void escapeJoinAll() throws Exception {
41 | String actual =
42 | ShellEscaper.escapeJoinAll(
43 | Arrays.asList("foo", "@echo:-", "100", "$US", "a b", "\"qu'ot'es\"", "\"quot\"", "\\"));
44 | assertThat(actual)
45 | .isEqualTo("foo @echo:- 100 '$US' 'a b' '\"qu'\\''ot'\\''es\"' '\"quot\"' '\\'");
46 | }
47 | }
48 |
--------------------------------------------------------------------------------
/third_party/remote-apis/BUILD.bazel:
--------------------------------------------------------------------------------
1 | load("@grpc-java//:java_grpc_library.bzl", "java_grpc_library")
2 |
3 | package(default_visibility = ["//visibility:public"])
4 |
5 | java_grpc_library(
6 | name = "build_bazel_remote_asset_v1_remote_asset_java_grpc",
7 | srcs = [
8 | "@bazel_remote_apis//build/bazel/remote/asset/v1:remote_asset_proto",
9 | ],
10 | deps = [
11 | "@bazel_remote_apis//build/bazel/remote/asset/v1:remote_asset_java_proto",
12 | ],
13 | )
14 |
15 | java_grpc_library(
16 | name = "build_bazel_remote_execution_v2_remote_execution_java_grpc",
17 | srcs = [
18 | "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_proto",
19 | ],
20 | deps = [
21 | "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_java_proto",
22 | ],
23 | )
24 |
--------------------------------------------------------------------------------