entry : value.getAllFields().entrySet()) {
29 | if (entry.getKey().getType() == Descriptors.FieldDescriptor.Type.MESSAGE ||
30 | entry.getKey().getType() == Descriptors.FieldDescriptor.Type.GROUP) {
31 | if (entry.getKey().isRepeated()) {
32 | //noinspection unchecked
33 | for (Message message : (Iterable) entry.getValue()) {
34 | if (!isValid(message, context)) return false;
35 | }
36 | } else {
37 | if (!isValid((Message) entry.getValue(), context)) return false;
38 | }
39 | }
40 | }
41 | }
42 | return true;
43 | }
44 | }
45 |
--------------------------------------------------------------------------------
/src/main/java/org/signal/storageservice/util/HostSupplier.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2025 Signal Messenger, LLC
3 | * SPDX-License-Identifier: AGPL-3.0-only
4 | */
5 |
6 | package org.signal.storageservice.util;
7 |
8 | import com.google.cloud.MetadataConfig;
9 | import java.net.InetAddress;
10 | import java.net.UnknownHostException;
11 | import java.util.Optional;
12 | import java.util.UUID;
13 | import org.apache.commons.lang3.StringUtils;
14 |
15 | /**
16 | * This class attempts to supply a meaningful host string for use in metrics and logging attribution. It prioritizes the
17 | * hostname value via {@link InetAddress#getHostName()}, but falls back to:
18 | *
19 | * - an instance ID from cloud provider metadata
20 | * - random ID if no cloud provider instance ID is available
21 | *
22 | *
23 | * In the current implementation, only GCP is supported, but support for other
24 | * platforms may be added in the future.
25 | */
26 | public class HostSupplier {
27 |
28 | private static final String FALLBACK_INSTANCE_ID = UUID.randomUUID().toString();
29 |
30 | public static String getHost() {
31 | return getHostName()
32 | .orElse(StringUtils.defaultIfBlank(MetadataConfig.getInstanceId(), FALLBACK_INSTANCE_ID));
33 | }
34 |
35 | private static Optional getHostName() {
36 | try {
37 | final String hostname = InetAddress.getLocalHost().getHostName();
38 | if ("localhost".equals(hostname)) {
39 | return Optional.empty();
40 | }
41 |
42 | return Optional.ofNullable(hostname);
43 | } catch (UnknownHostException e) {
44 | return Optional.empty();
45 | }
46 | }
47 | }
48 |
--------------------------------------------------------------------------------
/src/main/java/org/signal/storageservice/providers/InvalidProtocolBufferExceptionMapper.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright © 2019 Smoke Turner, LLC (github@smoketurner.com)
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package org.signal.storageservice.providers;
17 |
18 | import com.google.protobuf.InvalidProtocolBufferException;
19 | import org.slf4j.Logger;
20 | import org.slf4j.LoggerFactory;
21 |
22 | import jakarta.ws.rs.core.MediaType;
23 | import jakarta.ws.rs.core.Response;
24 | import jakarta.ws.rs.ext.ExceptionMapper;
25 | import jakarta.ws.rs.ext.Provider;
26 |
27 | @Provider
28 | public class InvalidProtocolBufferExceptionMapper implements ExceptionMapper {
29 |
30 | private static final Logger LOGGER = LoggerFactory.getLogger(InvalidProtocolBufferExceptionMapper.class);
31 |
32 | @Override
33 | public Response toResponse(InvalidProtocolBufferException exception) {
34 | LOGGER.debug("Unable to process protocol buffer message", exception);
35 | return Response.status(Response.Status.BAD_REQUEST)
36 | .type(MediaType.TEXT_PLAIN_TYPE)
37 | .entity("Unable to process protocol buffer message")
38 | .build();
39 | }
40 | }
41 |
--------------------------------------------------------------------------------
/src/main/java/org/signal/storageservice/providers/ProtocolBufferValidationErrorMessageBodyWriter.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2020 Signal Messenger, LLC
3 | * SPDX-License-Identifier: AGPL-3.0-only
4 | */
5 |
6 | package org.signal.storageservice.providers;
7 |
8 | import io.dropwizard.jersey.validation.ValidationErrorMessage;
9 |
10 | import jakarta.ws.rs.Produces;
11 | import jakarta.ws.rs.WebApplicationException;
12 | import jakarta.ws.rs.core.MediaType;
13 | import jakarta.ws.rs.core.MultivaluedMap;
14 | import jakarta.ws.rs.ext.MessageBodyWriter;
15 | import jakarta.ws.rs.ext.Provider;
16 | import java.io.IOException;
17 | import java.io.OutputStream;
18 | import java.lang.annotation.Annotation;
19 | import java.lang.reflect.Type;
20 |
21 | @Provider
22 | @Produces({
23 | ProtocolBufferMediaType.APPLICATION_PROTOBUF,
24 | ProtocolBufferMediaType.APPLICATION_PROTOBUF_TEXT,
25 | ProtocolBufferMediaType.APPLICATION_PROTOBUF_JSON
26 | })
27 | public class ProtocolBufferValidationErrorMessageBodyWriter implements MessageBodyWriter {
28 | @Override
29 | public boolean isWriteable(Class type, Type genericType, Annotation[] annotations, MediaType mediaType) {
30 | return ValidationErrorMessage.class.isAssignableFrom(type);
31 | }
32 |
33 | @Override
34 | public long getSize(ValidationErrorMessage validationErrorMessage, Class type, Type genericType, Annotation[] annotations, MediaType mediaType) {
35 | return 0;
36 | }
37 |
38 | @Override
39 | public void writeTo(ValidationErrorMessage validationErrorMessage, Class type, Type genericType, Annotation[] annotations, MediaType mediaType, MultivaluedMap httpHeaders, OutputStream entityStream) throws IOException, WebApplicationException {
40 | }
41 | }
42 |
--------------------------------------------------------------------------------
/src/main/java/org/signal/storageservice/providers/ProtocolBufferMediaType.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright © 2019 Smoke Turner, LLC (github@smoketurner.com)
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package org.signal.storageservice.providers;
17 |
18 | import jakarta.ws.rs.core.MediaType;
19 |
20 | public class ProtocolBufferMediaType extends MediaType {
21 |
22 | /** "application/x-protobuf" */
23 | public static final String APPLICATION_PROTOBUF = "application/x-protobuf";
24 | /** "application/x-protobuf" */
25 | public static final MediaType APPLICATION_PROTOBUF_TYPE =
26 | new MediaType("application", "x-protobuf");
27 |
28 | /** "application/x-protobuf-text-format" */
29 | public static final String APPLICATION_PROTOBUF_TEXT = "application/x-protobuf-text-format";
30 | /** "application/x-protobuf-text-format" */
31 | public static final MediaType APPLICATION_PROTOBUF_TEXT_TYPE =
32 | new MediaType("application", "x-protobuf-text-format");
33 |
34 | /** "application/x-protobuf-json-format" */
35 | public static final String APPLICATION_PROTOBUF_JSON = "application/x-protobuf-json-format";
36 | /** "application/x-protobuf-json-format" */
37 | public static final MediaType APPLICATION_PROTOBUF_JSON_TYPE =
38 | new MediaType("application", "x-protobuf-json-format");
39 | }
40 |
--------------------------------------------------------------------------------
/src/main/java/org/signal/storageservice/filters/TimestampResponseFilter.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2025 Signal Messenger, LLC
3 | * SPDX-License-Identifier: AGPL-3.0-only
4 | */
5 |
6 | package org.signal.storageservice.filters;
7 |
8 | import java.io.IOException;
9 | import java.time.Clock;
10 | import jakarta.servlet.Filter;
11 | import jakarta.servlet.FilterChain;
12 | import jakarta.servlet.ServletException;
13 | import jakarta.servlet.ServletRequest;
14 | import jakarta.servlet.ServletResponse;
15 | import jakarta.servlet.http.HttpServletResponse;
16 | import jakarta.ws.rs.container.ContainerRequestContext;
17 | import jakarta.ws.rs.container.ContainerResponseContext;
18 | import jakarta.ws.rs.container.ContainerResponseFilter;
19 | import org.signal.storageservice.util.HeaderUtils;
20 |
21 | /**
22 | * Injects a timestamp header into all outbound responses.
23 | */
24 | public class TimestampResponseFilter implements Filter, ContainerResponseFilter {
25 |
26 | private final Clock clock;
27 |
28 | public TimestampResponseFilter(final Clock clock) {
29 | this.clock = clock;
30 | }
31 |
32 | @Override
33 | public void doFilter(final ServletRequest request, final ServletResponse response, final FilterChain chain)
34 | throws ServletException, IOException {
35 |
36 | if (response instanceof HttpServletResponse httpServletResponse) {
37 | httpServletResponse.setHeader(HeaderUtils.TIMESTAMP_HEADER, String.valueOf(clock.millis()));
38 | }
39 |
40 | chain.doFilter(request, response);
41 | }
42 |
43 | @Override
44 | public void filter(ContainerRequestContext requestContext, ContainerResponseContext responseContext) {
45 | // not using add() - it's ok to overwrite any existing header, and we don't want a multi-value
46 | responseContext.getHeaders().putSingle(HeaderUtils.TIMESTAMP_HEADER, String.valueOf(clock.millis()));
47 | }
48 | }
49 |
--------------------------------------------------------------------------------
/src/main/java/org/signal/storageservice/s3/PolicySigner.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2020 Signal Messenger, LLC
3 | * SPDX-License-Identifier: AGPL-3.0-only
4 | */
5 |
6 | package org.signal.storageservice.s3;
7 |
8 | import javax.crypto.Mac;
9 | import javax.crypto.spec.SecretKeySpec;
10 | import java.nio.charset.StandardCharsets;
11 | import java.security.InvalidKeyException;
12 | import java.security.NoSuchAlgorithmException;
13 | import java.time.ZonedDateTime;
14 | import java.time.format.DateTimeFormatter;
15 |
16 | public class PolicySigner {
17 |
18 | private final String awsAccessSecret;
19 | private final String region;
20 |
21 | public PolicySigner(String awsAccessSecret, String region) {
22 | this.awsAccessSecret = awsAccessSecret;
23 | this.region = region;
24 | }
25 |
26 | public String getSignature(ZonedDateTime now, String policy) {
27 | try {
28 | Mac mac = Mac.getInstance("HmacSHA256");
29 |
30 | mac.init(new SecretKeySpec(("AWS4" + awsAccessSecret).getBytes(StandardCharsets.UTF_8), "HmacSHA256"));
31 | byte[] dateKey = mac.doFinal(now.format(DateTimeFormatter.ofPattern("yyyyMMdd")).getBytes(StandardCharsets.UTF_8));
32 |
33 | mac.init(new SecretKeySpec(dateKey, "HmacSHA256"));
34 | byte[] dateRegionKey = mac.doFinal(region.getBytes(StandardCharsets.UTF_8));
35 |
36 | mac.init(new SecretKeySpec(dateRegionKey, "HmacSHA256"));
37 | byte[] dateRegionServiceKey = mac.doFinal("s3".getBytes(StandardCharsets.UTF_8));
38 |
39 | mac.init(new SecretKeySpec(dateRegionServiceKey, "HmacSHA256"));
40 | byte[] signingKey = mac.doFinal("aws4_request".getBytes(StandardCharsets.UTF_8));
41 |
42 | mac.init(new SecretKeySpec(signingKey, "HmacSHA256"));
43 |
44 | return Base16Lower.encode(mac.doFinal(policy.getBytes(StandardCharsets.UTF_8)));
45 | } catch (NoSuchAlgorithmException | InvalidKeyException e) {
46 | throw new AssertionError(e);
47 | }
48 | }
49 |
50 | }
51 |
--------------------------------------------------------------------------------
/src/main/java/org/signal/storageservice/metrics/UserAgentTagUtil.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2013-2020 Signal Messenger, LLC
3 | * SPDX-License-Identifier: AGPL-3.0-only
4 | */
5 |
6 | package org.signal.storageservice.metrics;
7 |
8 | import com.vdurmont.semver4j.Semver;
9 | import io.micrometer.core.instrument.Tag;
10 | import org.signal.storageservice.util.ua.ClientPlatform;
11 | import org.signal.storageservice.util.ua.UnrecognizedUserAgentException;
12 | import org.signal.storageservice.util.ua.UserAgent;
13 | import org.signal.storageservice.util.ua.UserAgentUtil;
14 | import java.util.Collections;
15 | import java.util.Map;
16 | import java.util.Optional;
17 | import java.util.Set;
18 |
19 | /**
20 | * Utility class for extracting platform/version metrics tags from User-Agent strings.
21 | */
22 | public class UserAgentTagUtil {
23 |
24 | public static final String PLATFORM_TAG = "platform";
25 | public static final String VERSION_TAG = "clientVersion";
26 |
27 | private UserAgentTagUtil() {
28 | }
29 |
30 | public static Tag getPlatformTag(final String userAgentString) {
31 | String platform;
32 |
33 | try {
34 | platform = UserAgentUtil.parseUserAgentString(userAgentString).platform().name().toLowerCase();
35 | } catch (final UnrecognizedUserAgentException e) {
36 | platform = "unrecognized";
37 | }
38 |
39 | return Tag.of(PLATFORM_TAG, platform);
40 | }
41 |
42 | public static Optional getClientVersionTag(final String userAgentString,
43 | final Map> recognizedVersionsByPlatform) {
44 |
45 | try {
46 | final UserAgent userAgent = UserAgentUtil.parseUserAgentString(userAgentString);
47 |
48 | final Set recognizedVersions =
49 | recognizedVersionsByPlatform.getOrDefault(userAgent.platform(), Collections.emptySet());
50 |
51 | if (recognizedVersions.contains(userAgent.version())) {
52 | return Optional.of(Tag.of(VERSION_TAG, userAgent.version().toString()));
53 | }
54 | } catch (final UnrecognizedUserAgentException ignored) {
55 | }
56 |
57 | return Optional.empty();
58 | }
59 | }
60 |
--------------------------------------------------------------------------------
/src/test/java/org/signal/storageservice/auth/GroupUserTest.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2013-2022 Signal Messenger, LLC
3 | * SPDX-License-Identifier: AGPL-3.0-only
4 | */
5 |
6 | package org.signal.storageservice.auth;
7 |
8 | import static org.junit.jupiter.api.Assertions.assertEquals;
9 |
10 | import java.util.Random;
11 | import java.util.stream.Stream;
12 |
13 | import org.junit.jupiter.params.ParameterizedTest;
14 | import org.junit.jupiter.params.provider.Arguments;
15 | import org.junit.jupiter.params.provider.MethodSource;
16 |
17 | import com.google.protobuf.ByteString;
18 |
19 | class GroupUserTest {
20 |
21 | @ParameterizedTest
22 | @MethodSource
23 | void isMember(final ByteString userAci,
24 | final ByteString userPni,
25 | final ByteString userPublicKey,
26 | final ByteString memberUuid,
27 | final ByteString groupPublicKey,
28 | final boolean expectIsMember) {
29 |
30 | final GroupUser groupUser = new GroupUser(userAci, userPni, userPublicKey, generateRandomByteString());
31 |
32 | assertEquals(expectIsMember, groupUser.isMember(memberUuid, groupPublicKey));
33 | }
34 |
35 | private static Stream isMember() {
36 | final ByteString memberUuid = generateRandomByteString();
37 | final ByteString groupPublicKey = generateRandomByteString();
38 |
39 | return Stream.of(
40 | Arguments.of(memberUuid, generateRandomByteString(), groupPublicKey, memberUuid, groupPublicKey, true),
41 | Arguments.of(memberUuid, null, groupPublicKey, memberUuid, groupPublicKey, true),
42 | Arguments.of(generateRandomByteString(), memberUuid, groupPublicKey, memberUuid, groupPublicKey, true),
43 | Arguments.of(generateRandomByteString(), null, groupPublicKey, memberUuid, groupPublicKey, false),
44 | Arguments.of(generateRandomByteString(), generateRandomByteString(), groupPublicKey, memberUuid, groupPublicKey,
45 | false),
46 | Arguments.of(memberUuid, null, generateRandomByteString(), memberUuid, groupPublicKey, false));
47 | }
48 |
49 | private static ByteString generateRandomByteString() {
50 | final Random random = new Random();
51 | final byte[] bytes = new byte[16];
52 |
53 | random.nextBytes(bytes);
54 |
55 | return ByteString.copyFrom(bytes);
56 | }
57 | }
58 |
--------------------------------------------------------------------------------
/src/main/java/org/signal/storageservice/auth/ExternalGroupCredentialGenerator.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2020 Signal Messenger, LLC
3 | * SPDX-License-Identifier: AGPL-3.0-only
4 | */
5 |
6 | package org.signal.storageservice.auth;
7 |
8 | import com.google.protobuf.ByteString;
9 | import org.apache.commons.codec.binary.Hex;
10 | import org.signal.storageservice.util.Util;
11 |
12 | import javax.crypto.Mac;
13 | import javax.crypto.spec.SecretKeySpec;
14 | import java.nio.charset.StandardCharsets;
15 | import java.security.InvalidKeyException;
16 | import java.security.MessageDigest;
17 | import java.security.NoSuchAlgorithmException;
18 | import java.time.Clock;
19 |
20 | public class ExternalGroupCredentialGenerator {
21 |
22 | private final byte[] key;
23 | private final Clock clock;
24 |
25 | public ExternalGroupCredentialGenerator(byte[] key, Clock clock) {
26 | this.key = key;
27 | this.clock = clock;
28 | }
29 |
30 | public String generateFor(ByteString uuidCiphertext, ByteString groupId, boolean isAllowedToInitiateGroupCall) {
31 | final MessageDigest digest = getDigestInstance();
32 | final long currentTimeSeconds = clock.millis() / 1000;
33 | String encodedData =
34 | "2:"
35 | + Hex.encodeHexString(digest.digest(uuidCiphertext.toByteArray())) + ":"
36 | + Hex.encodeHexString(groupId.toByteArray()) + ":"
37 | + currentTimeSeconds + ":"
38 | + (isAllowedToInitiateGroupCall ? "1" : "0");
39 | String truncatedHmac = Hex.encodeHexString(
40 | Util.truncate(getHmac(key, encodedData.getBytes(StandardCharsets.UTF_8)), 10));
41 |
42 | return encodedData + ":" + truncatedHmac;
43 | }
44 |
45 | private Mac getMacInstance() {
46 | try {
47 | return Mac.getInstance("HmacSHA256");
48 | } catch (NoSuchAlgorithmException e) {
49 | throw new AssertionError(e);
50 | }
51 | }
52 |
53 | private MessageDigest getDigestInstance() {
54 | try {
55 | return MessageDigest.getInstance("SHA-256");
56 | } catch (NoSuchAlgorithmException e) {
57 | throw new AssertionError(e);
58 | }
59 | }
60 |
61 | private byte[] getHmac(byte[] key, byte[] input) {
62 | try {
63 | final Mac mac = getMacInstance();
64 | mac.init(new SecretKeySpec(key, "HmacSHA256"));
65 | return mac.doFinal(input);
66 | } catch (InvalidKeyException e) {
67 | throw new AssertionError(e);
68 | }
69 | }
70 | }
71 |
--------------------------------------------------------------------------------
/src/test/java/org/signal/storageservice/controllers/GroupsControllerPhoneNumberPrivacyTest.java:
--------------------------------------------------------------------------------
1 | package org.signal.storageservice.controllers;
2 |
3 | import jakarta.ws.rs.client.Entity;
4 | import jakarta.ws.rs.core.Response;
5 | import com.google.protobuf.ByteString;
6 | import org.junit.jupiter.api.Test;
7 | import org.signal.storageservice.providers.ProtocolBufferMediaType;
8 | import org.signal.storageservice.storage.protos.groups.Group;
9 | import org.signal.storageservice.storage.protos.groups.GroupChange;
10 | import org.signal.storageservice.storage.protos.groups.Member;
11 | import org.signal.storageservice.storage.protos.groups.MemberPendingProfileKey;
12 | import org.signal.storageservice.util.AuthHelper;
13 |
14 | class GroupsControllerPhoneNumberPrivacyTest extends BaseGroupsControllerTest {
15 |
16 | @Test
17 | void testRejectInvitationFromPni() throws Exception {
18 | Group.Builder groupBuilder = Group.newBuilder(this.group)
19 | .addMembersPendingProfileKey(MemberPendingProfileKey.newBuilder()
20 | .setMember(Member.newBuilder()
21 | .setUserId(validUserThreePniId)
22 | .setRole(Member.Role.DEFAULT)
23 | .setJoinedAtVersion(0)
24 | .build())
25 | .setAddedByUserId(validUserId)
26 | .setTimestamp(clock.millis())
27 | .build());
28 |
29 | setupGroupsManagerBehaviors(groupBuilder.build());
30 |
31 | GroupChange.Actions.Builder actionsBuilder = GroupChange.Actions.newBuilder()
32 | .setVersion(1)
33 | .addDeleteMembersPendingProfileKey(GroupChange.Actions.DeleteMemberPendingProfileKeyAction.newBuilder()
34 | .setDeletedUserId(validUserThreePniId)
35 | .build());
36 |
37 | groupBuilder.clearMembersPendingProfileKey().setVersion(1);
38 |
39 | try (Response response = resources.getJerseyTest()
40 | .target("/v1/groups/")
41 | .request(ProtocolBufferMediaType.APPLICATION_PROTOBUF)
42 | .header("Authorization", AuthHelper.getAuthHeader(groupSecretParams, AuthHelper.VALID_USER_THREE_AUTH_CREDENTIAL))
43 | .method("PATCH", Entity.entity(actionsBuilder.build().toByteArray(), ProtocolBufferMediaType.APPLICATION_PROTOBUF))) {
44 |
45 | actionsBuilder.setGroupId(ByteString.copyFrom(groupPublicParams.getGroupIdentifier().serialize()));
46 | verifyGroupModification(groupBuilder, actionsBuilder, 0, response, validUserThreePniId);
47 | }
48 | }
49 | }
50 |
--------------------------------------------------------------------------------
/src/test/java/org/signal/storageservice/filters/TimestampResponseFilterTest.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2025 Signal Messenger, LLC
3 | * SPDX-License-Identifier: AGPL-3.0-only
4 | */
5 |
6 | package org.signal.storageservice.filters;
7 |
8 | import static org.junit.jupiter.api.Assertions.assertEquals;
9 | import static org.junit.jupiter.api.Assertions.assertTrue;
10 | import static org.mockito.ArgumentMatchers.eq;
11 | import static org.mockito.Mockito.mock;
12 | import static org.mockito.Mockito.verify;
13 | import static org.mockito.Mockito.when;
14 |
15 | import java.time.Clock;
16 | import java.time.Instant;
17 | import java.time.ZoneId;
18 | import jakarta.servlet.FilterChain;
19 | import jakarta.servlet.http.HttpServletRequest;
20 | import jakarta.servlet.http.HttpServletResponse;
21 | import jakarta.ws.rs.container.ContainerRequestContext;
22 | import jakarta.ws.rs.container.ContainerResponseContext;
23 | import jakarta.ws.rs.core.MultivaluedMap;
24 | import org.junit.jupiter.api.Test;
25 | import org.signal.storageservice.util.HeaderUtils;
26 |
27 | class TimestampResponseFilterTest {
28 |
29 | private static final long EPOCH_MILLIS = 1738182156000L;
30 |
31 | private static final Clock CLOCK = Clock.fixed(Instant.ofEpochMilli(EPOCH_MILLIS), ZoneId.systemDefault());
32 |
33 | @Test
34 | void testJerseyFilter() {
35 | final ContainerRequestContext requestContext = mock(ContainerRequestContext.class);
36 | final ContainerResponseContext responseContext = mock(ContainerResponseContext.class);
37 | final MultivaluedMap headers = org.glassfish.jersey.message.internal.HeaderUtils.createOutbound();
38 | when(responseContext.getHeaders()).thenReturn(headers);
39 |
40 | new TimestampResponseFilter(CLOCK).filter(requestContext, responseContext);
41 |
42 | assertTrue(headers.containsKey(HeaderUtils.TIMESTAMP_HEADER));
43 | assertEquals(1, headers.get(HeaderUtils.TIMESTAMP_HEADER).size());
44 | assertEquals(String.valueOf(EPOCH_MILLIS), headers.get(HeaderUtils.TIMESTAMP_HEADER).get(0));
45 | }
46 |
47 | @Test
48 | void testServletFilter() throws Exception {
49 | final HttpServletRequest request = mock(HttpServletRequest.class);
50 | final HttpServletResponse response = mock(HttpServletResponse.class);
51 |
52 | new TimestampResponseFilter(CLOCK).doFilter(request, response, mock(FilterChain.class));
53 |
54 | verify(response).setHeader(eq(HeaderUtils.TIMESTAMP_HEADER), eq(String.valueOf(EPOCH_MILLIS)));
55 | }
56 | }
57 |
--------------------------------------------------------------------------------
/src/test/java/org/signal/storageservice/util/TestClock.java:
--------------------------------------------------------------------------------
1 | package org.signal.storageservice.util;
2 |
3 | import java.time.Clock;
4 | import java.time.Instant;
5 | import java.time.ZoneId;
6 | import java.util.Optional;
7 |
8 | /**
9 | * Clock class specialized for testing.
10 | *
11 | * This clock can be pinned to a particular instant or can provide the "normal" time.
12 | *
13 | * Unlike normal clocks it can be dynamically pinned and unpinned to help with testing.
14 | * It should not be used in production.
15 | */
16 | public class TestClock extends Clock {
17 |
18 | private volatile Optional pinnedInstant;
19 | private final ZoneId zoneId;
20 |
21 | private TestClock(Optional maybePinned, ZoneId id) {
22 | this.pinnedInstant = maybePinned;
23 | this.zoneId = id;
24 | }
25 |
26 | /**
27 | * Instantiate a test clock that returns the "real" time.
28 | *
29 | * The clock can later be pinned to an instant if desired.
30 | *
31 | * @return unpinned test clock.
32 | */
33 | public static TestClock now() {
34 | return new TestClock(Optional.empty(), ZoneId.of("UTC"));
35 | }
36 |
37 | /**
38 | * Instantiate a test clock pinned to a particular instant.
39 | *
40 | * The clock can later be pinned to a different instant or unpinned if desired.
41 | *
42 | * Unlike the fixed constructor no time zone is required (it defaults to UTC).
43 | *
44 | * @param instant the instant to pin the clock to.
45 | * @return test clock pinned to the given instant.
46 | */
47 | public static TestClock pinned(Instant instant) {
48 | return new TestClock(Optional.of(instant), ZoneId.of("UTC"));
49 | }
50 |
51 | /**
52 | * Pin this test clock to the given instance.
53 | *
54 | * This modifies the existing clock in-place.
55 | *
56 | * @param instant the instant to pin the clock to.
57 | */
58 | public void pin(Instant instant) {
59 | this.pinnedInstant = Optional.of(instant);
60 | }
61 |
62 | /**
63 | * Unpin this test clock so it will being returning the "real" time.
64 | *
65 | * This modifies the existing clock in-place.
66 | */
67 | public void unpin() {
68 | this.pinnedInstant = Optional.empty();
69 | }
70 |
71 |
72 | @Override
73 | public TestClock withZone(ZoneId id) {
74 | return new TestClock(pinnedInstant, id);
75 | }
76 |
77 | @Override
78 | public ZoneId getZone() {
79 | return zoneId;
80 | }
81 |
82 | @Override
83 | public Instant instant() {
84 | return pinnedInstant.orElseGet(Instant::now);
85 | }
86 |
87 | }
88 |
--------------------------------------------------------------------------------
/src/main/java/org/signal/storageservice/s3/PostPolicyGenerator.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2020 Signal Messenger, LLC
3 | * SPDX-License-Identifier: AGPL-3.0-only
4 | */
5 |
6 | package org.signal.storageservice.s3;
7 |
8 | import org.apache.commons.codec.binary.Base64;
9 | import org.signal.storageservice.util.Pair;
10 |
11 | import java.nio.charset.StandardCharsets;
12 | import java.time.ZonedDateTime;
13 | import java.time.format.DateTimeFormatter;
14 |
15 | public class PostPolicyGenerator {
16 |
17 | public static final DateTimeFormatter AWS_DATE_TIME = DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmssX");
18 | private static final DateTimeFormatter CREDENTIAL_DATE = DateTimeFormatter.ofPattern("yyyyMMdd" );
19 |
20 | private final String region;
21 | private final String bucket;
22 | private final String awsAccessId;
23 |
24 | public PostPolicyGenerator(String region, String bucket, String awsAccessId) {
25 | this.region = region;
26 | this.bucket = bucket;
27 | this.awsAccessId = awsAccessId;
28 | }
29 |
30 | public Pair createFor(ZonedDateTime now, String object, int maxSizeInBytes) {
31 | String expiration = now.plusMinutes(30).format(DateTimeFormatter.ISO_INSTANT);
32 | String credentialDate = now.format(CREDENTIAL_DATE);
33 | String requestDate = now.format(AWS_DATE_TIME );
34 | String credential = String.format("%s/%s/%s/s3/aws4_request", awsAccessId, credentialDate, region);
35 |
36 | String policy = String.format("{ \"expiration\": \"%s\",\n" +
37 | " \"conditions\": [\n" +
38 | " {\"bucket\": \"%s\"},\n" +
39 | " {\"key\": \"%s\"},\n" +
40 | " {\"acl\": \"private\"},\n" +
41 | " [\"starts-with\", \"$Content-Type\", \"\"],\n" +
42 | " [\"content-length-range\", 1, " + maxSizeInBytes + "],\n" +
43 | "\n" +
44 | " {\"x-amz-credential\": \"%s\"},\n" +
45 | " {\"x-amz-algorithm\": \"AWS4-HMAC-SHA256\"},\n" +
46 | " {\"x-amz-date\": \"%s\" }\n" +
47 | " ]\n" +
48 | "}", expiration, bucket, object, credential, requestDate);
49 |
50 | return new Pair<>(credential, Base64.encodeBase64String(policy.getBytes(StandardCharsets.UTF_8)));
51 | }
52 |
53 | }
54 |
--------------------------------------------------------------------------------
/src/main/java/org/signal/storageservice/auth/GroupUserAuthenticator.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2020 Signal Messenger, LLC
3 | * SPDX-License-Identifier: AGPL-3.0-only
4 | */
5 |
6 | package org.signal.storageservice.auth;
7 |
8 | import static com.codahale.metrics.MetricRegistry.name;
9 |
10 | import com.google.protobuf.ByteString;
11 | import io.dropwizard.auth.Authenticator;
12 | import io.dropwizard.auth.basic.BasicCredentials;
13 | import io.micrometer.core.instrument.Metrics;
14 | import java.util.Optional;
15 | import org.apache.commons.codec.DecoderException;
16 | import org.apache.commons.codec.binary.Hex;
17 | import org.signal.libsignal.zkgroup.InvalidInputException;
18 | import org.signal.libsignal.zkgroup.VerificationFailedException;
19 | import org.signal.libsignal.zkgroup.auth.AuthCredentialPresentation;
20 | import org.signal.libsignal.zkgroup.auth.ServerZkAuthOperations;
21 | import org.signal.libsignal.zkgroup.groups.GroupPublicParams;
22 |
23 | public class GroupUserAuthenticator implements Authenticator {
24 |
25 | private static final String CREDENTIALS_VERSION_COUNTER_NAME = name(GroupUserAuthenticator.class,
26 | "credentialsVersion");
27 |
28 | private final ServerZkAuthOperations serverZkAuthOperations;
29 |
30 | public GroupUserAuthenticator(ServerZkAuthOperations serverZkAuthOperations) {
31 | this.serverZkAuthOperations = serverZkAuthOperations;
32 | }
33 |
34 | @Override
35 | public Optional authenticate(BasicCredentials basicCredentials) {
36 | try {
37 | String encodedGroupPublicKey = basicCredentials.getUsername();
38 | String encodedPresentation = basicCredentials.getPassword();
39 |
40 | GroupPublicParams groupPublicKey = new GroupPublicParams(Hex.decodeHex(encodedGroupPublicKey));
41 | AuthCredentialPresentation presentation = new AuthCredentialPresentation(Hex.decodeHex(encodedPresentation));
42 |
43 | Metrics.counter(CREDENTIALS_VERSION_COUNTER_NAME, "credentialsVersion", presentation.getVersion().toString())
44 | .increment();
45 |
46 | serverZkAuthOperations.verifyAuthCredentialPresentation(groupPublicKey, presentation);
47 |
48 | return Optional.of(new GroupUser(ByteString.copyFrom(presentation.getUuidCiphertext().serialize()),
49 | presentation.getPniCiphertext() != null ? ByteString.copyFrom(presentation.getPniCiphertext().serialize()) : null,
50 | ByteString.copyFrom(groupPublicKey.serialize()),
51 | ByteString.copyFrom(groupPublicKey.getGroupIdentifier().serialize())));
52 |
53 | } catch (DecoderException | VerificationFailedException | InvalidInputException e) {
54 | return Optional.empty();
55 | }
56 | }
57 | }
58 |
--------------------------------------------------------------------------------
/src/main/java/org/signal/storageservice/StorageServiceConfiguration.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2020 Signal Messenger, LLC
3 | * SPDX-License-Identifier: AGPL-3.0-only
4 | */
5 |
6 | package org.signal.storageservice;
7 |
8 | import com.fasterxml.jackson.annotation.JsonProperty;
9 | import com.vdurmont.semver4j.Semver;
10 | import io.dropwizard.core.Configuration;
11 | import org.signal.storageservice.configuration.AuthenticationConfiguration;
12 | import org.signal.storageservice.configuration.BigTableConfiguration;
13 | import org.signal.storageservice.configuration.CdnConfiguration;
14 | import org.signal.storageservice.configuration.DatadogConfiguration;
15 | import org.signal.storageservice.configuration.GroupConfiguration;
16 | import org.signal.storageservice.configuration.WarmupConfiguration;
17 | import org.signal.storageservice.configuration.ZkConfiguration;
18 |
19 | import jakarta.validation.Valid;
20 | import jakarta.validation.constraints.NotNull;
21 | import org.signal.storageservice.util.ua.ClientPlatform;
22 | import java.util.Collections;
23 | import java.util.Map;
24 | import java.util.Set;
25 |
26 | public class StorageServiceConfiguration extends Configuration {
27 |
28 | @JsonProperty
29 | @Valid
30 | @NotNull
31 | private BigTableConfiguration bigtable;
32 |
33 | @JsonProperty
34 | @Valid
35 | @NotNull
36 | private AuthenticationConfiguration authentication;
37 |
38 | @JsonProperty
39 | @Valid
40 | @NotNull
41 | private ZkConfiguration zkConfig;
42 |
43 | @JsonProperty
44 | @Valid
45 | @NotNull
46 | private CdnConfiguration cdn;
47 |
48 | @JsonProperty
49 | @Valid
50 | @NotNull
51 | private GroupConfiguration group;
52 |
53 | @JsonProperty
54 | @Valid
55 | @NotNull
56 | private DatadogConfiguration datadog;
57 |
58 | @JsonProperty
59 | @Valid
60 | @NotNull
61 | private WarmupConfiguration warmup = new WarmupConfiguration(5);
62 |
63 | @JsonProperty
64 | @NotNull
65 | private Map> recognizedClientVersions = Collections.emptyMap();
66 |
67 | public BigTableConfiguration getBigTableConfiguration() {
68 | return bigtable;
69 | }
70 |
71 | public AuthenticationConfiguration getAuthenticationConfiguration() {
72 | return authentication;
73 | }
74 |
75 | public ZkConfiguration getZkConfiguration() {
76 | return zkConfig;
77 | }
78 |
79 | public CdnConfiguration getCdnConfiguration() {
80 | return cdn;
81 | }
82 |
83 | public GroupConfiguration getGroupConfiguration() {
84 | return group;
85 | }
86 |
87 | public DatadogConfiguration getDatadogConfiguration() {
88 | return datadog;
89 | }
90 |
91 | public WarmupConfiguration getWarmUpConfiguration() {
92 | return warmup;
93 | }
94 |
95 | public Map> getRecognizedClientVersions() {
96 | return recognizedClientVersions;
97 | }
98 | }
99 |
--------------------------------------------------------------------------------
/src/main/java/org/signal/storageservice/auth/GroupUser.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2020 Signal Messenger, LLC
3 | * SPDX-License-Identifier: AGPL-3.0-only
4 | */
5 |
6 | package org.signal.storageservice.auth;
7 |
8 | import com.google.common.annotations.VisibleForTesting;
9 | import com.google.protobuf.ByteString;
10 | import org.signal.storageservice.storage.protos.groups.Member;
11 | import org.signal.libsignal.zkgroup.InvalidInputException;
12 | import org.signal.libsignal.zkgroup.groups.GroupPublicParams;
13 |
14 | import javax.annotation.Nullable;
15 | import javax.security.auth.Subject;
16 | import java.security.MessageDigest;
17 | import java.security.Principal;
18 | import java.util.Optional;
19 |
20 | public class GroupUser implements Principal {
21 |
22 | private final ByteString aciCiphertext;
23 | private final ByteString groupPublicKey;
24 | private final ByteString groupId;
25 |
26 | @Nullable
27 | private final ByteString pniCiphertext;
28 |
29 | public GroupUser(ByteString aciCiphertext, @Nullable ByteString pniCiphertext, ByteString groupPublicKey, ByteString groupId) {
30 | this.aciCiphertext = aciCiphertext;
31 | this.pniCiphertext = pniCiphertext;
32 | this.groupPublicKey = groupPublicKey;
33 | this.groupId = groupId;
34 | }
35 |
36 | public boolean isMember(Member member, ByteString groupPublicKey) {
37 | return isMember(member.getUserId(), groupPublicKey);
38 | }
39 |
40 | public boolean aciMatches(ByteString uuid) {
41 | return MessageDigest.isEqual(this.aciCiphertext.toByteArray(), uuid.toByteArray());
42 | }
43 |
44 | public boolean isMember(ByteString uuid, ByteString groupPublicKey) {
45 | final boolean publicKeyMatches = MessageDigest.isEqual(this.groupPublicKey.toByteArray(), groupPublicKey.toByteArray());
46 | final boolean aciMatches = MessageDigest.isEqual(this.aciCiphertext.toByteArray(), uuid.toByteArray());
47 | final boolean pniMatches =
48 | pniCiphertext != null && MessageDigest.isEqual(this.pniCiphertext.toByteArray(), uuid.toByteArray());
49 |
50 | return publicKeyMatches && (aciMatches || pniMatches);
51 | }
52 |
53 | public GroupPublicParams getGroupPublicKey() {
54 | try {
55 | return new GroupPublicParams(groupPublicKey.toByteArray());
56 | } catch (InvalidInputException e) {
57 | throw new AssertionError(e);
58 | }
59 | }
60 |
61 | public ByteString getGroupId() {
62 | return groupId;
63 | }
64 |
65 | @VisibleForTesting
66 | ByteString getAciCiphertext() {
67 | return aciCiphertext;
68 | }
69 |
70 | public Optional getPniCiphertext() {
71 | return Optional.ofNullable(pniCiphertext);
72 | }
73 |
74 | // Principal implementation
75 |
76 | @Override
77 | public String getName() {
78 | return null;
79 | }
80 |
81 | @Override
82 | public boolean implies(Subject subject) {
83 | return false;
84 | }
85 | }
86 |
--------------------------------------------------------------------------------
/src/main/java/org/signal/storageservice/auth/ExternalServiceCredentialValidator.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2020 Signal Messenger, LLC
3 | * SPDX-License-Identifier: AGPL-3.0-only
4 | */
5 |
6 | package org.signal.storageservice.auth;
7 |
8 | import org.apache.commons.codec.DecoderException;
9 | import org.apache.commons.codec.binary.Hex;
10 | import org.signal.storageservice.util.Util;
11 | import org.slf4j.Logger;
12 | import org.slf4j.LoggerFactory;
13 |
14 | import javax.crypto.Mac;
15 | import javax.crypto.spec.SecretKeySpec;
16 | import java.security.InvalidKeyException;
17 | import java.security.MessageDigest;
18 | import java.security.NoSuchAlgorithmException;
19 | import java.util.concurrent.TimeUnit;
20 |
21 | public class ExternalServiceCredentialValidator {
22 |
23 | private final Logger logger = LoggerFactory.getLogger(ExternalServiceCredentialValidator.class);
24 |
25 | private final byte[] key;
26 |
27 | public ExternalServiceCredentialValidator(byte[] key) {
28 | this.key = key;
29 | }
30 |
31 | public boolean isValid(String token, String number, long currentTimeMillis) {
32 | String[] parts = token.split(":");
33 | Mac mac = getMacInstance();
34 |
35 | if (parts.length != 3) {
36 | return false;
37 | }
38 |
39 | if (!number.equals(parts[0])) {
40 | return false;
41 | }
42 |
43 | if (!isValidTime(parts[1], currentTimeMillis)) {
44 | return false;
45 | }
46 |
47 | return isValidSignature(parts[0] + ":" + parts[1], parts[2], mac);
48 | }
49 |
50 | private boolean isValidTime(String timeString, long currentTimeMillis) {
51 | try {
52 | long tokenTime = Long.parseLong(timeString);
53 | long ourTime = TimeUnit.MILLISECONDS.toSeconds(currentTimeMillis);
54 |
55 | return TimeUnit.SECONDS.toHours(Math.abs(ourTime - tokenTime)) < 24;
56 | } catch (NumberFormatException e) {
57 | logger.warn("Number Format", e);
58 | return false;
59 | }
60 | }
61 |
62 | private boolean isValidSignature(String prefix, String suffix, Mac mac) {
63 | try {
64 | byte[] ourSuffix = Util.truncate(getHmac(key, prefix.getBytes(), mac), 10);
65 | byte[] theirSuffix = Hex.decodeHex(suffix.toCharArray());
66 |
67 | return MessageDigest.isEqual(ourSuffix, theirSuffix);
68 | } catch (DecoderException e) {
69 | logger.warn("DirectoryCredentials", e);
70 | return false;
71 | }
72 | }
73 |
74 | private Mac getMacInstance() {
75 | try {
76 | return Mac.getInstance("HmacSHA256");
77 | } catch (NoSuchAlgorithmException e) {
78 | throw new AssertionError(e);
79 | }
80 | }
81 |
82 | private byte[] getHmac(byte[] key, byte[] input, Mac mac) {
83 | try {
84 | mac.init(new SecretKeySpec(key, "HmacSHA256"));
85 | return mac.doFinal(input);
86 | } catch (InvalidKeyException e) {
87 | throw new AssertionError(e);
88 | }
89 | }
90 |
91 | }
92 |
--------------------------------------------------------------------------------
/src/test/java/org/signal/storageservice/auth/ExternalGroupCredentialGeneratorTest.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2020 Signal Messenger, LLC
3 | * SPDX-License-Identifier: AGPL-3.0-only
4 | */
5 |
6 | package org.signal.storageservice.auth;
7 |
8 | import static org.assertj.core.api.Assertions.assertThat;
9 | import static org.junit.jupiter.api.Assertions.assertArrayEquals;
10 | import static org.junit.jupiter.params.provider.Arguments.arguments;
11 | import static org.mockito.Mockito.mock;
12 | import static org.mockito.Mockito.when;
13 |
14 | import com.google.protobuf.ByteString;
15 | import java.nio.charset.StandardCharsets;
16 | import java.security.InvalidKeyException;
17 | import java.security.MessageDigest;
18 | import java.security.NoSuchAlgorithmException;
19 | import java.security.SecureRandom;
20 | import java.time.Clock;
21 | import java.util.stream.Stream;
22 | import javax.crypto.Mac;
23 | import javax.crypto.spec.SecretKeySpec;
24 | import org.apache.commons.codec.DecoderException;
25 | import org.apache.commons.codec.binary.Hex;
26 | import org.junit.jupiter.params.ParameterizedTest;
27 | import org.junit.jupiter.params.provider.Arguments;
28 | import org.junit.jupiter.params.provider.MethodSource;
29 | import org.signal.storageservice.util.Util;
30 |
31 | public class ExternalGroupCredentialGeneratorTest {
32 |
33 | @SuppressWarnings("unused")
34 | static Stream testArgumentsProvider() {
35 | return Stream.of(
36 | arguments(true, "1"),
37 | arguments(false, "0")
38 | );
39 | }
40 |
41 | @ParameterizedTest
42 | @MethodSource("testArgumentsProvider")
43 | public void testGenerateValidCredentials(boolean isAllowedToCreateGroupCalls, String part4)
44 | throws DecoderException, NoSuchAlgorithmException, InvalidKeyException {
45 | Clock clock = mock(Clock.class);
46 | final long timeInMillis = new SecureRandom().nextLong() & Long.MAX_VALUE;
47 | when(clock.millis()).thenReturn(timeInMillis);
48 |
49 | byte[] key = Util.generateSecretBytes(32);
50 | ExternalGroupCredentialGenerator generator = new ExternalGroupCredentialGenerator(key, clock);
51 | ByteString uuid = ByteString.copyFrom(Util.generateSecretBytes(16));
52 | ByteString groupId = ByteString.copyFrom(Util.generateSecretBytes(16));
53 |
54 | String token = generator.generateFor(uuid, groupId, isAllowedToCreateGroupCalls);
55 |
56 | String[] parts = token.split(":");
57 | assertThat(parts.length).isEqualTo(6);
58 |
59 | assertThat(parts[0]).isEqualTo("2");
60 | assertArrayEquals(Hex.decodeHex(parts[1]), MessageDigest.getInstance("SHA-256").digest(uuid.toByteArray()));
61 | assertArrayEquals(Hex.decodeHex(parts[2]), groupId.toByteArray());
62 | assertThat(Long.parseLong(parts[3])).isEqualTo(timeInMillis / 1000);
63 | assertThat(parts[4]).isEqualTo(part4);
64 |
65 | byte[] theirMac = Hex.decodeHex(parts[5]);
66 | Mac hmac = Mac.getInstance("HmacSHA256");
67 | hmac.init(new SecretKeySpec(key, "HmacSHA256"));
68 | byte[] ourMac = Util.truncate(hmac.doFinal(
69 | ("2:" + parts[1] + ":" + parts[2] + ":" + parts[3] + ":" + part4).getBytes(StandardCharsets.UTF_8)), 10);
70 | assertArrayEquals(ourMac, theirMac);
71 | }
72 | }
73 |
--------------------------------------------------------------------------------
/src/main/java/org/signal/storageservice/metrics/SignalDatadogReporterFactory.java:
--------------------------------------------------------------------------------
1 | /*
2 | * This is derived from Coursera's dropwizard datadog reporter.
3 | * https://github.com/coursera/metrics-datadog
4 | */
5 |
6 | package org.signal.storageservice.metrics;
7 |
8 | import com.codahale.metrics.MetricRegistry;
9 | import com.codahale.metrics.ScheduledReporter;
10 | import com.fasterxml.jackson.annotation.JsonProperty;
11 | import com.fasterxml.jackson.annotation.JsonTypeName;
12 | import io.dropwizard.metrics.common.BaseReporterFactory;
13 | import java.util.ArrayList;
14 | import java.util.EnumSet;
15 | import java.util.List;
16 | import jakarta.validation.Valid;
17 | import jakarta.validation.constraints.NotNull;
18 | import org.coursera.metrics.datadog.DatadogReporter;
19 | import org.coursera.metrics.datadog.DatadogReporter.Expansion;
20 | import org.coursera.metrics.datadog.DefaultMetricNameFormatterFactory;
21 | import org.coursera.metrics.datadog.DynamicTagsCallbackFactory;
22 | import org.coursera.metrics.datadog.MetricNameFormatterFactory;
23 | import org.coursera.metrics.datadog.transport.AbstractTransportFactory;
24 | import org.signal.storageservice.StorageServiceVersion;
25 | import org.signal.storageservice.util.HostSupplier;
26 |
27 | @JsonTypeName("signal-datadog")
28 | public class SignalDatadogReporterFactory extends BaseReporterFactory {
29 |
30 | @JsonProperty
31 | private String host = null;
32 |
33 | @JsonProperty
34 | private List tags = null;
35 |
36 | @Valid
37 | @JsonProperty
38 | private DynamicTagsCallbackFactory dynamicTagsCallback = null;
39 |
40 | @JsonProperty
41 | private String prefix = null;
42 |
43 | @Valid
44 | @NotNull
45 | @JsonProperty
46 | private EnumSet expansions = EnumSet.of(
47 | Expansion.COUNT,
48 | Expansion.MIN,
49 | Expansion.MAX,
50 | Expansion.MEAN,
51 | Expansion.MEDIAN,
52 | Expansion.P95,
53 | Expansion.P99
54 | );
55 |
56 | @Valid
57 | @NotNull
58 | @JsonProperty
59 | private MetricNameFormatterFactory metricNameFormatter = new DefaultMetricNameFormatterFactory();
60 |
61 | @Valid
62 | @NotNull
63 | @JsonProperty
64 | private AbstractTransportFactory transport = null;
65 |
66 | @Override
67 | public ScheduledReporter build(final MetricRegistry registry) {
68 |
69 | final List combinedTags;
70 | final String versionTag = "version:" + StorageServiceVersion.getServiceVersion();
71 |
72 | if (tags != null) {
73 | combinedTags = new ArrayList<>(tags);
74 | combinedTags.add(versionTag);
75 | } else {
76 | combinedTags = new ArrayList<>((List.of(versionTag)));
77 | }
78 |
79 | return DatadogReporter.forRegistry(registry)
80 | .withTransport(transport.build())
81 | .withHost(HostSupplier.getHost())
82 | .withTags(combinedTags)
83 | .withPrefix(prefix)
84 | .withExpansions(expansions)
85 | .withMetricNameFormatter(metricNameFormatter.build())
86 | .withDynamicTagCallback(dynamicTagsCallback != null ? dynamicTagsCallback.build() : null)
87 | .filter(getFilter())
88 | .convertDurationsTo(getDurationUnit())
89 | .convertRatesTo(getRateUnit())
90 | .build();
91 | }
92 | }
93 |
--------------------------------------------------------------------------------
/src/test/java/org/signal/storageservice/util/ua/UserAgentUtilTest.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2013-2022 Signal Messenger, LLC
3 | * SPDX-License-Identifier: AGPL-3.0-only
4 | */
5 |
6 | package org.signal.storageservice.util.ua;
7 |
8 | import static org.junit.jupiter.api.Assertions.assertEquals;
9 | import static org.junit.jupiter.api.Assertions.assertThrows;
10 |
11 | import com.vdurmont.semver4j.Semver;
12 | import java.util.stream.Stream;
13 | import javax.annotation.Nullable;
14 | import org.junit.jupiter.params.ParameterizedTest;
15 | import org.junit.jupiter.params.provider.Arguments;
16 | import org.junit.jupiter.params.provider.MethodSource;
17 |
18 | class UserAgentUtilTest {
19 |
20 | @ParameterizedTest
21 | @MethodSource("argumentsForTestParseStandardUserAgentString")
22 | void testParseStandardUserAgentString(final String userAgentString, @Nullable final UserAgent expectedUserAgent)
23 | throws UnrecognizedUserAgentException {
24 |
25 | if (expectedUserAgent != null) {
26 | assertEquals(expectedUserAgent, UserAgentUtil.parseUserAgentString(userAgentString));
27 | } else {
28 | assertThrows(UnrecognizedUserAgentException.class, () -> UserAgentUtil.parseUserAgentString(userAgentString));
29 | }
30 | }
31 |
32 | private static Stream argumentsForTestParseStandardUserAgentString() {
33 | return Stream.of(
34 | Arguments.of("This is obviously not a reasonable User-Agent string.", null),
35 | Arguments.of("Signal-Android/4.68.3 Android/25",
36 | new UserAgent(ClientPlatform.ANDROID, new Semver("4.68.3"), "Android/25")),
37 | Arguments.of("Signal-Android/4.68.3", new UserAgent(ClientPlatform.ANDROID, new Semver("4.68.3"), null)),
38 | Arguments.of("Signal-Desktop/1.2.3 Linux", new UserAgent(ClientPlatform.DESKTOP, new Semver("1.2.3"), "Linux")),
39 | Arguments.of("Signal-Desktop/1.2.3 macOS", new UserAgent(ClientPlatform.DESKTOP, new Semver("1.2.3"), "macOS")),
40 | Arguments.of("Signal-Desktop/1.2.3 Windows",
41 | new UserAgent(ClientPlatform.DESKTOP, new Semver("1.2.3"), "Windows")),
42 | Arguments.of("Signal-Desktop/1.2.3", new UserAgent(ClientPlatform.DESKTOP, new Semver("1.2.3"), null)),
43 | Arguments.of("Signal-Desktop/1.32.0-beta.3",
44 | new UserAgent(ClientPlatform.DESKTOP, new Semver("1.32.0-beta.3"), null)),
45 | Arguments.of("Signal-iOS/3.9.0 (iPhone; iOS 12.2; Scale/3.00)",
46 | new UserAgent(ClientPlatform.IOS, new Semver("3.9.0"), "(iPhone; iOS 12.2; Scale/3.00)")),
47 | Arguments.of("Signal-iOS/3.9.0 iOS/14.2", new UserAgent(ClientPlatform.IOS, new Semver("3.9.0"), "iOS/14.2")),
48 | Arguments.of("Signal-iOS/3.9.0", new UserAgent(ClientPlatform.IOS, new Semver("3.9.0"), null)),
49 | Arguments.of("Signal-Android/7.11.23-nightly-1982-06-28-07-07-07 tonic/0.31",
50 | new UserAgent(ClientPlatform.ANDROID, new Semver("7.11.23-nightly-1982-06-28-07-07-07"), "tonic/0.31")),
51 | Arguments.of("Signal-Android/7.11.23-nightly-1982-06-28-07-07-07 Android/42 tonic/0.31",
52 | new UserAgent(ClientPlatform.ANDROID, new Semver("7.11.23-nightly-1982-06-28-07-07-07"), "Android/42 tonic/0.31")),
53 | Arguments.of("Signal-Android/7.6.2 Android/34 libsignal/0.46.0",
54 | new UserAgent(ClientPlatform.ANDROID, new Semver("7.6.2"), "Android/34 libsignal/0.46.0")));
55 | }
56 | }
57 |
--------------------------------------------------------------------------------
/src/main/java/org/signal/storageservice/storage/GroupsTable.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2020 Signal Messenger, LLC
3 | * SPDX-License-Identifier: AGPL-3.0-only
4 | */
5 |
6 | package org.signal.storageservice.storage;
7 |
8 | import com.codahale.metrics.MetricRegistry;
9 | import com.codahale.metrics.SharedMetricRegistries;
10 | import com.codahale.metrics.Timer;
11 | import com.google.cloud.bigtable.data.v2.BigtableDataClient;
12 | import com.google.cloud.bigtable.data.v2.models.Mutation;
13 | import com.google.protobuf.ByteString;
14 | import com.google.protobuf.InvalidProtocolBufferException;
15 | import org.signal.storageservice.metrics.StorageMetrics;
16 | import org.signal.storageservice.storage.protos.groups.Group;
17 |
18 | import java.util.Optional;
19 | import java.util.concurrent.CompletableFuture;
20 |
21 | import static com.codahale.metrics.MetricRegistry.name;
22 |
23 | public class GroupsTable extends Table {
24 |
25 | public static final String FAMILY = "g";
26 |
27 | public static final String COLUMN_GROUP_DATA = "gr";
28 | public static final String COLUMN_VERSION = "ver";
29 |
30 | private final MetricRegistry metricRegistry = SharedMetricRegistries.getOrCreate(StorageMetrics.NAME);
31 | private final Timer getTimer = metricRegistry.timer(name(GroupsTable.class, "get" ));
32 | private final Timer createTimer = metricRegistry.timer(name(GroupsTable.class, "create"));
33 | private final Timer updateTimer = metricRegistry.timer(name(GroupsTable.class, "update"));
34 |
35 | public GroupsTable(BigtableDataClient client, String tableId) {
36 | super(client, tableId);
37 | }
38 |
39 | public CompletableFuture> getGroup(ByteString groupId) {
40 | return toFuture(client.readRowAsync(tableId, groupId), getTimer).thenApply(row -> {
41 | if (row == null) return Optional.empty();
42 |
43 | try {
44 | ByteString groupData = row.getCells(FAMILY, COLUMN_GROUP_DATA)
45 | .stream()
46 | .filter(cell -> cell.getTimestamp() == 0)
47 | .findFirst()
48 | .orElseThrow()
49 | .getValue();
50 |
51 | return Optional.of(Group.parseFrom(groupData));
52 | } catch (InvalidProtocolBufferException e) {
53 | throw new AssertionError(e);
54 | }
55 | });
56 | }
57 |
58 | public CompletableFuture createGroup(ByteString groupId, Group group) {
59 | Mutation mutation = Mutation.create()
60 | .setCell(FAMILY, ByteString.copyFromUtf8(COLUMN_GROUP_DATA), 0, group.toByteString())
61 | .setCell(FAMILY, COLUMN_VERSION, 0, String.valueOf(group.getVersion()));
62 |
63 | return setIfEmpty(createTimer, groupId, FAMILY, COLUMN_GROUP_DATA, mutation);
64 | }
65 |
66 | public CompletableFuture updateGroup(ByteString groupId, Group group) {
67 | Mutation mutation = Mutation.create()
68 | .setCell(FAMILY, ByteString.copyFromUtf8(COLUMN_GROUP_DATA), 0, group.toByteString())
69 | .setCell(FAMILY, COLUMN_VERSION, 0, String.valueOf(group.getVersion()));
70 |
71 | return setIfValue(updateTimer, groupId, FAMILY, COLUMN_VERSION, String.valueOf(group.getVersion() - 1), mutation);
72 | }
73 |
74 | }
75 |
--------------------------------------------------------------------------------
/src/test/java/org/signal/storageservice/metrics/UserAgentTagUtilTest.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2025 Signal Messenger, LLC
3 | * SPDX-License-Identifier: AGPL-3.0-only
4 | */
5 |
6 | package org.signal.storageservice.metrics;
7 |
8 | import static org.junit.jupiter.api.Assertions.assertEquals;
9 |
10 | import com.vdurmont.semver4j.Semver;
11 | import io.micrometer.core.instrument.Tag;
12 | import java.util.Map;
13 | import java.util.Optional;
14 | import java.util.Set;
15 | import java.util.stream.Stream;
16 | import org.junit.jupiter.api.Test;
17 | import org.junit.jupiter.params.ParameterizedTest;
18 | import org.junit.jupiter.params.provider.Arguments;
19 | import org.junit.jupiter.params.provider.MethodSource;
20 | import org.signal.storageservice.util.ua.ClientPlatform;
21 |
22 | class UserAgentTagUtilTest {
23 |
24 | @ParameterizedTest
25 | @MethodSource
26 | void getPlatformTag(final String userAgent, final Tag expectedTag) {
27 | assertEquals(expectedTag, UserAgentTagUtil.getPlatformTag(userAgent));
28 | }
29 |
30 | private static Stream getPlatformTag() {
31 | return Stream.of(
32 | Arguments.of("This is obviously not a reasonable User-Agent string.", Tag.of(UserAgentTagUtil.PLATFORM_TAG, "unrecognized")),
33 | Arguments.of(null, Tag.of(UserAgentTagUtil.PLATFORM_TAG, "unrecognized")),
34 | Arguments.of("Signal-Android/4.53.7 (Android 8.1)", Tag.of(UserAgentTagUtil.PLATFORM_TAG, "android")),
35 | Arguments.of("Signal-Desktop/1.2.3", Tag.of(UserAgentTagUtil.PLATFORM_TAG, "desktop")),
36 | Arguments.of("Signal-iOS/3.9.0 (iPhone; iOS 12.2; Scale/3.00)", Tag.of(UserAgentTagUtil.PLATFORM_TAG, "ios")),
37 | Arguments.of("Signal-Android/1.2.3 (Android 8.1)", Tag.of(UserAgentTagUtil.PLATFORM_TAG, "android")),
38 | Arguments.of("Signal-Desktop/3.9.0", Tag.of(UserAgentTagUtil.PLATFORM_TAG, "desktop")),
39 | Arguments.of("Signal-iOS/4.53.7 (iPhone; iOS 12.2; Scale/3.00)", Tag.of(UserAgentTagUtil.PLATFORM_TAG, "ios")),
40 | Arguments.of("Signal-Android/4.68.3 (Android 9)", Tag.of(UserAgentTagUtil.PLATFORM_TAG, "android")),
41 | Arguments.of("Signal-Android/1.2.3 (Android 4.3)", Tag.of(UserAgentTagUtil.PLATFORM_TAG, "android")),
42 | Arguments.of("Signal-Android/4.68.3.0-bobsbootlegclient", Tag.of(UserAgentTagUtil.PLATFORM_TAG, "android")),
43 | Arguments.of("Signal-Desktop/1.22.45-foo-0", Tag.of(UserAgentTagUtil.PLATFORM_TAG, "desktop")),
44 | Arguments.of("Signal-Desktop/1.34.5-beta.1-fakeclientemporium", Tag.of(UserAgentTagUtil.PLATFORM_TAG, "desktop")),
45 | Arguments.of("Signal-Desktop/1.32.0-beta.3", Tag.of(UserAgentTagUtil.PLATFORM_TAG, "desktop"))
46 | );
47 | }
48 |
49 | @Test
50 | void getClientVersionTag() {
51 | final Map> recognizedClientVersions =
52 | Map.of(ClientPlatform.ANDROID, Set.of(new Semver("1.2.3")));
53 |
54 | assertEquals(Optional.of(Tag.of(UserAgentTagUtil.VERSION_TAG, "1.2.3")),
55 | UserAgentTagUtil.getClientVersionTag("Signal-Android/1.2.3 (Android 4.3)", recognizedClientVersions));
56 |
57 | assertEquals(Optional.empty(),
58 | UserAgentTagUtil.getClientVersionTag("Signal-Android/1.2.4 (Android 4.3)", recognizedClientVersions));
59 |
60 | assertEquals(Optional.empty(),
61 | UserAgentTagUtil.getClientVersionTag("Signal-Desktop/1.22.45-foo-0", recognizedClientVersions));
62 | }
63 | }
64 |
--------------------------------------------------------------------------------
/src/main/java/org/signal/storageservice/storage/GroupsManager.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2020 Signal Messenger, LLC
3 | * SPDX-License-Identifier: AGPL-3.0-only
4 | */
5 |
6 | package org.signal.storageservice.storage;
7 |
8 | import com.google.cloud.bigtable.data.v2.BigtableDataClient;
9 | import com.google.protobuf.ByteString;
10 | import java.util.List;
11 | import java.util.Optional;
12 | import java.util.concurrent.CompletableFuture;
13 | import org.signal.storageservice.storage.protos.groups.Group;
14 | import org.signal.storageservice.storage.protos.groups.GroupChange;
15 | import org.signal.storageservice.storage.protos.groups.GroupChanges.GroupChangeState;
16 | import javax.annotation.Nullable;
17 |
18 | public class GroupsManager {
19 |
20 | private final GroupsTable groupsTable;
21 | private final GroupLogTable groupLogTable;
22 |
23 | public GroupsManager(BigtableDataClient client, String groupsTableId, String groupLogsTableId) {
24 | this.groupsTable = new GroupsTable (client, groupsTableId );
25 | this.groupLogTable = new GroupLogTable(client, groupLogsTableId);
26 | }
27 |
28 | public CompletableFuture> getGroup(ByteString groupId) {
29 | return groupsTable.getGroup(groupId);
30 | }
31 |
32 | public CompletableFuture createGroup(ByteString groupId, Group group) {
33 | return groupsTable.createGroup(groupId, group);
34 | }
35 |
36 | public CompletableFuture> updateGroup(ByteString groupId, Group group) {
37 | return groupsTable.updateGroup(groupId, group)
38 | .thenCompose(modified -> {
39 | if (modified) return CompletableFuture.completedFuture(Optional.empty());
40 | else return getGroup(groupId).thenApply(result -> Optional.of(result.orElseThrow()));
41 | });
42 | }
43 |
44 | public CompletableFuture> getChangeRecords(ByteString groupId, Group group,
45 | @Nullable Integer maxSupportedChangeEpoch, boolean includeFirstState, boolean includeLastState,
46 | int fromVersionInclusive, int toVersionExclusive) {
47 | if (fromVersionInclusive >= toVersionExclusive) {
48 | throw new IllegalArgumentException("Version to read from (" + fromVersionInclusive + ") must be less than version to read to (" + toVersionExclusive + ")");
49 | }
50 |
51 | return groupLogTable.getRecordsFromVersion(groupId, maxSupportedChangeEpoch, includeFirstState, includeLastState, fromVersionInclusive, toVersionExclusive, group.getVersion())
52 | .thenApply(groupChangeStatesAndSeenCurrentVersion -> {
53 | List groupChangeStates = groupChangeStatesAndSeenCurrentVersion.first();
54 | boolean seenCurrentVersion = groupChangeStatesAndSeenCurrentVersion.second();
55 | if (isGroupInRange(group, fromVersionInclusive, toVersionExclusive) && !seenCurrentVersion && toVersionExclusive - 1 == group.getVersion()) {
56 | groupChangeStates.add(GroupChangeState.newBuilder().setGroupState(group).build());
57 | }
58 | return groupChangeStates;
59 | });
60 | }
61 |
62 | public CompletableFuture appendChangeRecord(ByteString groupId, int version, GroupChange change, Group state) {
63 | return groupLogTable.append(groupId, version, change, state);
64 | }
65 |
66 | private static boolean isGroupInRange(Group group, int fromVersionInclusive, int toVersionExclusive) {
67 | return fromVersionInclusive <= group.getVersion() && group.getVersion() < toVersionExclusive;
68 | }
69 | }
70 |
--------------------------------------------------------------------------------
/src/main/java/org/signal/storageservice/metrics/MetricsUtil.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2025 Signal Messenger, LLC
3 | * SPDX-License-Identifier: AGPL-3.0-only
4 | */
5 |
6 | package org.signal.storageservice.metrics;
7 |
8 | import com.codahale.metrics.SharedMetricRegistries;
9 | import io.dropwizard.core.setup.Environment;
10 | import io.micrometer.core.instrument.Metrics;
11 | import io.micrometer.core.instrument.Tags;
12 | import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics;
13 | import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics;
14 | import io.micrometer.core.instrument.binder.system.FileDescriptorMetrics;
15 | import io.micrometer.core.instrument.binder.system.ProcessorMetrics;
16 | import io.micrometer.datadog.DatadogMeterRegistry;
17 | import org.signal.storageservice.StorageServiceConfiguration;
18 | import org.signal.storageservice.StorageServiceVersion;
19 | import org.signal.storageservice.util.HostSupplier;
20 |
21 | public class MetricsUtil {
22 |
23 | public static final String PREFIX = "storage";
24 |
25 | private static volatile boolean registeredMetrics = false;
26 |
27 | /**
28 | * Returns a dot-separated ('.') name for the given class and name parts
29 | */
30 | public static String name(Class clazz, String... parts) {
31 | return name(clazz.getSimpleName(), parts);
32 | }
33 |
34 | private static String name(String name, String... parts) {
35 | final StringBuilder sb = new StringBuilder(PREFIX);
36 | sb.append(".").append(name);
37 | for (String part : parts) {
38 | sb.append(".").append(part);
39 | }
40 | return sb.toString();
41 | }
42 |
43 | public static void configureRegistries(final StorageServiceConfiguration config, final Environment environment) {
44 |
45 | if (registeredMetrics) {
46 | throw new IllegalStateException("Metric registries configured more than once");
47 | }
48 |
49 | registeredMetrics = true;
50 |
51 | SharedMetricRegistries.add(StorageMetrics.NAME, environment.metrics());
52 |
53 | {
54 | final DatadogMeterRegistry datadogMeterRegistry = new DatadogMeterRegistry(
55 | config.getDatadogConfiguration(), io.micrometer.core.instrument.Clock.SYSTEM);
56 |
57 | datadogMeterRegistry.config().commonTags(
58 | Tags.of(
59 | "service", "storage",
60 | "host", HostSupplier.getHost(),
61 | "version", StorageServiceVersion.getServiceVersion(),
62 | "env", config.getDatadogConfiguration().getEnvironment()));
63 |
64 | Metrics.addRegistry(datadogMeterRegistry);
65 | }
66 | }
67 |
68 |
69 | public static void registerSystemResourceMetrics(final Environment environment) {
70 | // Dropwizard metrics - some are temporarily duplicated for continuity
71 | environment.metrics().register(name(CpuUsageGauge.class, "cpu"), new CpuUsageGauge());
72 | environment.metrics().register(name(FreeMemoryGauge.class, "free_memory"), new FreeMemoryGauge());
73 | environment.metrics().register(name(NetworkSentGauge.class, "bytes_sent"), new NetworkSentGauge());
74 | environment.metrics().register(name(NetworkReceivedGauge.class, "bytes_received"), new NetworkReceivedGauge());
75 | environment.metrics().register(name(FileDescriptorGauge.class, "fd_count"), new FileDescriptorGauge());
76 |
77 | // Micrometer metrics
78 | new ProcessorMetrics().bindTo(Metrics.globalRegistry);
79 | new FreeMemoryGauge().bindTo(Metrics.globalRegistry);
80 | new FileDescriptorMetrics().bindTo(Metrics.globalRegistry);
81 |
82 | new JvmMemoryMetrics().bindTo(Metrics.globalRegistry);
83 | new JvmThreadMetrics().bindTo(Metrics.globalRegistry);
84 | }
85 |
86 | }
87 |
--------------------------------------------------------------------------------
/src/main/java/org/signal/storageservice/metrics/LogstashTcpSocketAppenderFactory.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2021 Signal Messenger, LLC
3 | * SPDX-License-Identifier: AGPL-3.0-only
4 | */
5 |
6 | package org.signal.storageservice.metrics;
7 |
8 | import ch.qos.logback.classic.LoggerContext;
9 | import ch.qos.logback.classic.PatternLayout;
10 | import ch.qos.logback.classic.spi.ILoggingEvent;
11 | import ch.qos.logback.core.Appender;
12 | import ch.qos.logback.core.encoder.LayoutWrappingEncoder;
13 | import ch.qos.logback.core.net.ssl.SSLConfiguration;
14 | import com.fasterxml.jackson.annotation.JsonProperty;
15 | import com.fasterxml.jackson.annotation.JsonTypeName;
16 | import com.fasterxml.jackson.databind.node.JsonNodeFactory;
17 | import com.fasterxml.jackson.databind.node.ObjectNode;
18 | import com.fasterxml.jackson.databind.node.TextNode;
19 | import io.dropwizard.logging.common.AbstractAppenderFactory;
20 | import io.dropwizard.logging.common.async.AsyncAppenderFactory;
21 | import io.dropwizard.logging.common.filter.LevelFilterFactory;
22 | import io.dropwizard.logging.common.layout.LayoutFactory;
23 | import java.time.Duration;
24 | import jakarta.validation.constraints.NotEmpty;
25 | import net.logstash.logback.appender.LogstashTcpSocketAppender;
26 | import net.logstash.logback.encoder.LogstashEncoder;
27 | import org.signal.storageservice.StorageServiceVersion;
28 | import org.signal.storageservice.util.HostSupplier;
29 |
30 | @JsonTypeName("logstashtcpsocket")
31 | public class LogstashTcpSocketAppenderFactory extends AbstractAppenderFactory {
32 |
33 | private String destination;
34 | private Duration keepAlive = Duration.ofSeconds(20);
35 | private String apiKey;
36 | private String environment;
37 |
38 | @JsonProperty
39 | @NotEmpty
40 | public String getDestination() {
41 | return destination;
42 | }
43 |
44 | @JsonProperty
45 | public Duration getKeepAlive() {
46 | return keepAlive;
47 | }
48 |
49 | @JsonProperty
50 | @NotEmpty
51 | public String getApiKey() {
52 | return apiKey;
53 | }
54 |
55 | @JsonProperty
56 | @NotEmpty
57 | public String getEnvironment() {
58 | return environment;
59 | }
60 |
61 | @Override
62 | public Appender build(
63 | final LoggerContext context,
64 | final String applicationName,
65 | final LayoutFactory layoutFactory,
66 | final LevelFilterFactory levelFilterFactory,
67 | final AsyncAppenderFactory asyncAppenderFactory) {
68 |
69 | final SSLConfiguration sslConfiguration = new SSLConfiguration();
70 | final LogstashTcpSocketAppender appender = new LogstashTcpSocketAppender();
71 | appender.setName("logstashtcpsocket-appender");
72 | appender.setContext(context);
73 | appender.setSsl(sslConfiguration);
74 | appender.addDestination(destination);
75 | appender.setKeepAliveDuration(new ch.qos.logback.core.util.Duration(keepAlive.toMillis()));
76 |
77 | final LogstashEncoder encoder = new LogstashEncoder();
78 | final ObjectNode customFieldsNode = new ObjectNode(JsonNodeFactory.instance);
79 | customFieldsNode.set("host", TextNode.valueOf(HostSupplier.getHost()));
80 | customFieldsNode.set("service", TextNode.valueOf("storage"));
81 | customFieldsNode.set("ddsource", TextNode.valueOf("logstash"));
82 | customFieldsNode.set("ddtags", TextNode.valueOf("env:" + environment + ",version:" + StorageServiceVersion.getServiceVersion()));
83 | encoder.setCustomFields(customFieldsNode.toString());
84 | final LayoutWrappingEncoder prefix = new LayoutWrappingEncoder<>();
85 | final PatternLayout layout = new PatternLayout();
86 | layout.setPattern(String.format("%s ", apiKey));
87 | prefix.setLayout(layout);
88 | encoder.setPrefix(prefix);
89 | appender.setEncoder(encoder);
90 |
91 | appender.addFilter(levelFilterFactory.build(threshold));
92 | getFilterFactories().forEach(f -> appender.addFilter(f.build()));
93 | appender.start();
94 |
95 | return wrapAsync(appender, asyncAppenderFactory);
96 | }
97 | }
98 |
--------------------------------------------------------------------------------
/src/test/java/org/signal/storageservice/auth/GroupUserAuthenticatorTest.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2013-2022 Signal Messenger, LLC
3 | * SPDX-License-Identifier: AGPL-3.0-only
4 | */
5 |
6 | package org.signal.storageservice.auth;
7 |
8 | import com.google.protobuf.ByteString;
9 | import io.dropwizard.auth.basic.BasicCredentials;
10 | import org.apache.commons.codec.binary.Hex;
11 | import org.junit.jupiter.api.BeforeEach;
12 | import org.junit.jupiter.api.Test;
13 | import org.signal.libsignal.protocol.ServiceId.Aci;
14 | import org.signal.libsignal.protocol.ServiceId.Pni;
15 | import org.signal.libsignal.zkgroup.ServerSecretParams;
16 | import org.signal.libsignal.zkgroup.auth.AuthCredentialPresentation;
17 | import org.signal.libsignal.zkgroup.auth.AuthCredentialWithPni;
18 | import org.signal.libsignal.zkgroup.auth.AuthCredentialWithPniResponse;
19 | import org.signal.libsignal.zkgroup.auth.ClientZkAuthOperations;
20 | import org.signal.libsignal.zkgroup.auth.ServerZkAuthOperations;
21 | import org.signal.libsignal.zkgroup.groups.GroupPublicParams;
22 | import org.signal.libsignal.zkgroup.groups.GroupSecretParams;
23 |
24 | import java.time.Instant;
25 | import java.time.temporal.ChronoUnit;
26 | import java.util.Optional;
27 | import java.util.UUID;
28 |
29 | import static org.junit.jupiter.api.Assertions.assertAll;
30 | import static org.junit.jupiter.api.Assertions.assertEquals;
31 | import static org.junit.jupiter.api.Assertions.assertTrue;
32 |
33 | class GroupUserAuthenticatorTest {
34 |
35 | private static final ServerSecretParams SERVER_SECRET_PARAMS = ServerSecretParams.generate();
36 |
37 | private static final GroupSecretParams GROUP_SECRET_PARAMS = GroupSecretParams.generate();
38 | private static final GroupPublicParams GROUP_PUBLIC_PARAMS = GROUP_SECRET_PARAMS.getPublicParams();
39 |
40 | private static final byte[] GROUP_ID = GROUP_PUBLIC_PARAMS.serialize();
41 | private static final byte[] GROUP_PUBLIC_KEY = GROUP_PUBLIC_PARAMS.serialize();
42 |
43 | private GroupUserAuthenticator groupUserAuthenticator;
44 |
45 | @BeforeEach
46 | void setUp() {
47 | groupUserAuthenticator = new GroupUserAuthenticator(new ServerZkAuthOperations(SERVER_SECRET_PARAMS));
48 | }
49 |
50 | @Test
51 | void authenticate() throws Exception {
52 | final Aci aci = new Aci(UUID.randomUUID());
53 | final Pni pni = new Pni(UUID.randomUUID());
54 |
55 | final Instant redemptionInstant = Instant.now().truncatedTo(ChronoUnit.DAYS);
56 |
57 | final ServerZkAuthOperations serverZkAuthOperations = new ServerZkAuthOperations(SERVER_SECRET_PARAMS);
58 | final ClientZkAuthOperations clientZkAuthOperations = new ClientZkAuthOperations(SERVER_SECRET_PARAMS.getPublicParams());
59 |
60 | final AuthCredentialWithPniResponse authCredentialWithPniResponse = serverZkAuthOperations.issueAuthCredentialWithPniZkc(aci, pni, redemptionInstant);
61 | final AuthCredentialWithPni authCredentialWithPni = clientZkAuthOperations.receiveAuthCredentialWithPniAsServiceId(aci, pni, redemptionInstant.getEpochSecond(), authCredentialWithPniResponse);
62 | final AuthCredentialPresentation authCredentialPresentation = clientZkAuthOperations.createAuthCredentialPresentation(GROUP_SECRET_PARAMS, authCredentialWithPni);
63 |
64 | final byte[] presentation = authCredentialPresentation.serialize();
65 | final GroupUser expectedGroupUser = new GroupUser(
66 | ByteString.copyFrom(authCredentialPresentation.getUuidCiphertext().serialize()),
67 | ByteString.copyFrom(authCredentialPresentation.getPniCiphertext().serialize()),
68 | ByteString.copyFrom(GROUP_PUBLIC_KEY),
69 | ByteString.copyFrom(GROUP_ID));
70 |
71 | final BasicCredentials basicCredentials =
72 | new BasicCredentials(Hex.encodeHexString(GROUP_PUBLIC_KEY), Hex.encodeHexString(presentation));
73 |
74 | final Optional maybeAuthenticatedUser = groupUserAuthenticator.authenticate(basicCredentials);
75 |
76 | assertTrue(maybeAuthenticatedUser.isPresent());
77 | assertGroupUserEqual(expectedGroupUser, maybeAuthenticatedUser.get());
78 | }
79 |
80 | private static void assertGroupUserEqual(final GroupUser expected, final GroupUser actual) {
81 | assertAll(
82 | () -> assertEquals(expected.getAciCiphertext(), actual.getAciCiphertext()),
83 | () -> assertEquals(expected.getPniCiphertext(), actual.getPniCiphertext())
84 | );
85 | }
86 | }
87 |
--------------------------------------------------------------------------------
/src/main/java/org/signal/storageservice/storage/StorageManager.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2020 Signal Messenger, LLC
3 | * SPDX-License-Identifier: AGPL-3.0-only
4 | */
5 |
6 | package org.signal.storageservice.storage;
7 |
8 | import com.google.cloud.bigtable.data.v2.BigtableDataClient;
9 | import com.google.protobuf.ByteString;
10 | import org.signal.storageservice.auth.User;
11 | import org.signal.storageservice.storage.protos.contacts.StorageItem;
12 | import org.signal.storageservice.storage.protos.contacts.StorageManifest;
13 | import org.slf4j.Logger;
14 | import org.slf4j.LoggerFactory;
15 |
16 | import java.util.List;
17 | import java.util.Optional;
18 | import java.util.concurrent.CompletableFuture;
19 |
20 | public class StorageManager {
21 |
22 | private final StorageManifestsTable manifestsTable;
23 | private final StorageItemsTable itemsTable;
24 |
25 | private static final Logger log = LoggerFactory.getLogger(StorageManager.class);
26 |
27 | public StorageManager(BigtableDataClient client, String contactManifestsTableId, String contactsTableId) {
28 | this.manifestsTable = new StorageManifestsTable(client, contactManifestsTableId);
29 | this.itemsTable = new StorageItemsTable(client, contactsTableId);
30 | }
31 |
32 | /**
33 | * Updates a manifest and applies mutations to stored items.
34 | *
35 | * @param user the user for whom to update manifests and mutate stored items
36 | * @param manifest the new manifest to store
37 | * @param inserts a list of new items to store
38 | * @param deletes a list of item identifiers to delete
39 | *
40 | * @return a future that completes when all updates and mutations have been applied; the future yields an empty value
41 | * if all updates and mutations were applied successfully, or the latest stored version of the {@code StorageManifest}
42 | * if the given {@code manifest}'s version is not exactly one version ahead of the stored manifest
43 | *
44 | * @see StorageManifestsTable#set(User, StorageManifest)
45 | */
46 | public CompletableFuture> set(User user, StorageManifest manifest, List inserts, List deletes) {
47 | return manifestsTable.set(user, manifest)
48 | .thenCompose(manifestUpdated -> {
49 | if (manifestUpdated) {
50 | return inserts.isEmpty() && deletes.isEmpty()
51 | ? CompletableFuture.completedFuture(Optional.empty())
52 | : itemsTable.set(user, inserts, deletes).thenApply(nothing -> Optional.empty());
53 | } else {
54 | // The new manifest's version wasn't the expected value, and it's likely that the manifest
55 | // was updated by a separate thread/process. Return a copy of the most recent stored
56 | // manifest.
57 | return getManifest(user).thenApply(retrieved -> Optional.of(retrieved.orElseThrow()));
58 | }
59 | });
60 | }
61 |
62 | public CompletableFuture> getManifest(User user) {
63 | return manifestsTable.get(user);
64 | }
65 |
66 | public CompletableFuture> getManifestIfNotVersion(User user, long version) {
67 | return manifestsTable.getIfNotVersion(user, version);
68 | }
69 |
70 | public CompletableFuture> getItems(User user, List keys) {
71 | return itemsTable.get(user, keys);
72 | }
73 |
74 | public CompletableFuture clearItems(User user) {
75 | return itemsTable.clear(user).whenComplete((ignored, throwable) -> {
76 | if (throwable != null) {
77 | log.warn("Failed to clear stored items", throwable);
78 | }
79 | });
80 | }
81 |
82 | public CompletableFuture delete(User user) {
83 | return CompletableFuture.allOf(
84 | itemsTable.clear(user).whenComplete((ignored, throwable) -> {
85 | if (throwable != null) {
86 | log.warn("Failed to delete stored items", throwable);
87 | }
88 | }),
89 |
90 | manifestsTable.clear(user).whenComplete((ignored, throwable) -> {
91 | if (throwable != null) {
92 | log.warn("Failed to delete manifest", throwable);
93 | }
94 | }));
95 | }
96 | }
97 |
--------------------------------------------------------------------------------
/src/main/java/org/signal/storageservice/util/Util.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2020 Signal Messenger, LLC
3 | * SPDX-License-Identifier: AGPL-3.0-only
4 | */
5 |
6 | package org.signal.storageservice.util;
7 |
8 | import java.io.UnsupportedEncodingException;
9 | import java.net.URLEncoder;
10 | import java.security.SecureRandom;
11 | import java.util.Arrays;
12 | import java.util.Map;
13 | import java.util.concurrent.TimeUnit;
14 |
15 | public class Util {
16 |
17 | public static int currentDaysSinceEpoch() {
18 | return Util.toIntExact(System.currentTimeMillis() / 1000 / 60 / 60 / 24);
19 | }
20 |
21 | public static int toIntExact(long value) {
22 | if ((int)value != value) {
23 | throw new ArithmeticException("integer overflow");
24 | }
25 | return (int)value;
26 | }
27 |
28 | public static String encodeFormParams(Map params) {
29 | try {
30 | StringBuffer buffer = new StringBuffer();
31 |
32 | for (String key : params.keySet()) {
33 | buffer.append(String.format("%s=%s",
34 | URLEncoder.encode(key, "UTF-8"),
35 | URLEncoder.encode(params.get(key), "UTF-8")));
36 | buffer.append("&");
37 | }
38 |
39 | buffer.deleteCharAt(buffer.length()-1);
40 | return buffer.toString();
41 | } catch (UnsupportedEncodingException e) {
42 | throw new AssertionError(e);
43 | }
44 | }
45 |
46 | public static boolean isEmpty(String param) {
47 | return param == null || param.length() == 0;
48 | }
49 |
50 | public static byte[] combine(byte[] one, byte[] two, byte[] three, byte[] four) {
51 | byte[] combined = new byte[one.length + two.length + three.length + four.length];
52 | System.arraycopy(one, 0, combined, 0, one.length);
53 | System.arraycopy(two, 0, combined, one.length, two.length);
54 | System.arraycopy(three, 0, combined, one.length + two.length, three.length);
55 | System.arraycopy(four, 0, combined, one.length + two.length + three.length, four.length);
56 |
57 | return combined;
58 | }
59 |
60 | public static byte[] truncate(byte[] element, int length) {
61 | byte[] result = new byte[length];
62 | System.arraycopy(element, 0, result, 0, result.length);
63 |
64 | return result;
65 | }
66 |
67 |
68 | public static byte[][] split(byte[] input, int firstLength, int secondLength) {
69 | byte[][] parts = new byte[2][];
70 |
71 | parts[0] = new byte[firstLength];
72 | System.arraycopy(input, 0, parts[0], 0, firstLength);
73 |
74 | parts[1] = new byte[secondLength];
75 | System.arraycopy(input, firstLength, parts[1], 0, secondLength);
76 |
77 | return parts;
78 | }
79 |
80 | public static byte[][] split(byte[] input, int firstLength, int secondLength, int thirdLength, int fourthLength) {
81 | byte[][] parts = new byte[4][];
82 |
83 | parts[0] = new byte[firstLength];
84 | System.arraycopy(input, 0, parts[0], 0, firstLength);
85 |
86 | parts[1] = new byte[secondLength];
87 | System.arraycopy(input, firstLength, parts[1], 0, secondLength);
88 |
89 | parts[2] = new byte[thirdLength];
90 | System.arraycopy(input, firstLength + secondLength, parts[2], 0, thirdLength);
91 |
92 | parts[3] = new byte[fourthLength];
93 | System.arraycopy(input, firstLength + secondLength + thirdLength, parts[3], 0, fourthLength);
94 |
95 | return parts;
96 | }
97 |
98 | public static byte[] generateSecretBytes(int size) {
99 | byte[] data = new byte[size];
100 | new SecureRandom().nextBytes(data);
101 | return data;
102 | }
103 |
104 | public static void sleep(long i) {
105 | try {
106 | Thread.sleep(i);
107 | } catch (InterruptedException ie) {}
108 | }
109 |
110 | public static void wait(Object object) {
111 | try {
112 | object.wait();
113 | } catch (InterruptedException e) {
114 | throw new AssertionError(e);
115 | }
116 | }
117 |
118 | public static void wait(Object object, long timeoutMs) {
119 | try {
120 | object.wait(timeoutMs);
121 | } catch (InterruptedException e) {
122 | throw new AssertionError(e);
123 | }
124 | }
125 |
126 | public static int hashCode(Object... objects) {
127 | return Arrays.hashCode(objects);
128 | }
129 |
130 | public static boolean isEquals(Object first, Object second) {
131 | return (first == null && second == null) || (first == second) || (first != null && first.equals(second));
132 | }
133 |
134 | public static long todayInMillis() {
135 | return TimeUnit.DAYS.toMillis(TimeUnit.MILLISECONDS.toDays(System.currentTimeMillis()));
136 | }
137 | }
138 |
--------------------------------------------------------------------------------
/src/test/java/org/signal/storageservice/metrics/MetricsHttpChannelListenerTest.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2024 Signal Messenger, LLC
3 | * SPDX-License-Identifier: AGPL-3.0-only
4 | */
5 |
6 | package org.signal.storageservice.metrics;
7 |
8 | import static org.junit.jupiter.api.Assertions.assertEquals;
9 | import static org.junit.jupiter.api.Assertions.assertTrue;
10 | import static org.mockito.ArgumentMatchers.any;
11 | import static org.mockito.ArgumentMatchers.eq;
12 | import static org.mockito.Mockito.mock;
13 | import static org.mockito.Mockito.verify;
14 | import static org.mockito.Mockito.when;
15 |
16 | import com.google.common.net.HttpHeaders;
17 | import io.micrometer.core.instrument.Counter;
18 | import io.micrometer.core.instrument.MeterRegistry;
19 | import io.micrometer.core.instrument.Tag;
20 | import java.util.HashSet;
21 | import java.util.List;
22 | import java.util.Set;
23 | import org.eclipse.jetty.http.HttpURI;
24 | import org.eclipse.jetty.server.Request;
25 | import org.eclipse.jetty.server.Response;
26 | import org.glassfish.jersey.server.ExtendedUriInfo;
27 | import org.glassfish.jersey.uri.UriTemplate;
28 | import org.junit.jupiter.api.BeforeEach;
29 | import org.junit.jupiter.api.Test;
30 | import org.mockito.ArgumentCaptor;
31 |
32 | class MetricsHttpChannelListenerTest {
33 |
34 | private MeterRegistry meterRegistry;
35 | private Counter requestCounter;
36 | private Counter requestBytesCounter;
37 | private Counter responseBytesCounter;
38 | private MetricsHttpChannelListener listener;
39 |
40 | @BeforeEach
41 | void setup() {
42 | meterRegistry = mock(MeterRegistry.class);
43 | requestCounter = mock(Counter.class);
44 | requestBytesCounter = mock(Counter.class);
45 | responseBytesCounter = mock(Counter.class);
46 |
47 | //noinspection unchecked
48 | when(meterRegistry.counter(eq(MetricsHttpChannelListener.REQUEST_COUNTER_NAME), any(Iterable.class)))
49 | .thenReturn(requestCounter);
50 |
51 | //noinspection unchecked
52 | when(meterRegistry.counter(eq(MetricsHttpChannelListener.REQUEST_BYTES_COUNTER_NAME), any(Iterable.class)))
53 | .thenReturn(requestBytesCounter);
54 |
55 | //noinspection unchecked
56 | when(meterRegistry.counter(eq(MetricsHttpChannelListener.RESPONSE_BYTES_COUNTER_NAME), any(Iterable.class)))
57 | .thenReturn(responseBytesCounter);
58 |
59 | listener = new MetricsHttpChannelListener(meterRegistry);
60 | }
61 |
62 | @Test
63 | @SuppressWarnings("unchecked")
64 | void testRequests() {
65 | final String path = "/test";
66 | final String method = "GET";
67 | final int statusCode = 200;
68 | final long requestContentLength = 5;
69 | final long responseContentLength = 7;
70 |
71 | final HttpURI httpUri = mock(HttpURI.class);
72 | when(httpUri.getPath()).thenReturn(path);
73 |
74 | final Request request = mock(Request.class);
75 | when(request.getMethod()).thenReturn(method);
76 | when(request.getHeader(HttpHeaders.USER_AGENT)).thenReturn("Signal-Android/4.53.7 (Android 8.1)");
77 | when(request.getHttpURI()).thenReturn(httpUri);
78 | when(request.getContentRead()).thenReturn(requestContentLength);
79 |
80 | final Response response = mock(Response.class);
81 | when(response.getStatus()).thenReturn(statusCode);
82 | when(response.getContentCount()).thenReturn(responseContentLength);
83 | when(request.getResponse()).thenReturn(response);
84 | final ExtendedUriInfo extendedUriInfo = mock(ExtendedUriInfo.class);
85 | when(request.getAttribute(MetricsHttpChannelListener.URI_INFO_PROPERTY_NAME)).thenReturn(extendedUriInfo);
86 | when(extendedUriInfo.getMatchedTemplates()).thenReturn(List.of(new UriTemplate(path)));
87 |
88 | final ArgumentCaptor> tagCaptor = ArgumentCaptor.forClass(Iterable.class);
89 |
90 | listener.onComplete(request);
91 |
92 | verify(requestCounter).increment();
93 | verify(requestBytesCounter).increment(requestContentLength);
94 | verify(responseBytesCounter).increment(responseContentLength);
95 |
96 | verify(meterRegistry).counter(eq(MetricsHttpChannelListener.REQUEST_COUNTER_NAME), tagCaptor.capture());
97 |
98 | final Set tags = new HashSet<>();
99 | for (final Tag tag : tagCaptor.getValue()) {
100 | tags.add(tag);
101 | }
102 |
103 | assertEquals(4, tags.size());
104 | assertTrue(tags.contains(Tag.of(MetricsHttpChannelListener.PATH_TAG, path)));
105 | assertTrue(tags.contains(Tag.of(MetricsHttpChannelListener.METHOD_TAG, method)));
106 | assertTrue(tags.contains(Tag.of(MetricsHttpChannelListener.STATUS_CODE_TAG, String.valueOf(statusCode))));
107 | assertTrue(tags.contains(Tag.of(UserAgentTagUtil.PLATFORM_TAG, "android")));
108 | }
109 | }
110 |
--------------------------------------------------------------------------------
/src/main/java/org/signal/storageservice/controllers/GroupsV1Controller.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2024 Signal Messenger, LLC
3 | * SPDX-License-Identifier: AGPL-3.0-only
4 | */
5 |
6 | package org.signal.storageservice.controllers;
7 |
8 | import com.codahale.metrics.annotation.Timed;
9 | import io.dropwizard.auth.Auth;
10 | import java.time.Clock;
11 | import java.time.Instant;
12 | import java.util.Optional;
13 | import java.util.concurrent.CompletableFuture;
14 | import jakarta.ws.rs.Consumes;
15 | import jakarta.ws.rs.DefaultValue;
16 | import jakarta.ws.rs.GET;
17 | import jakarta.ws.rs.HeaderParam;
18 | import jakarta.ws.rs.PATCH;
19 | import jakarta.ws.rs.PUT;
20 | import jakarta.ws.rs.Path;
21 | import jakarta.ws.rs.PathParam;
22 | import jakarta.ws.rs.Produces;
23 | import jakarta.ws.rs.QueryParam;
24 | import jakarta.ws.rs.core.Response;
25 | import org.signal.libsignal.zkgroup.ServerSecretParams;
26 | import org.signal.storageservice.auth.ExternalGroupCredentialGenerator;
27 | import org.signal.storageservice.auth.GroupUser;
28 | import org.signal.storageservice.configuration.GroupConfiguration;
29 | import org.signal.storageservice.providers.NoUnknownFields;
30 | import org.signal.storageservice.providers.ProtocolBufferMediaType;
31 | import org.signal.storageservice.s3.PolicySigner;
32 | import org.signal.storageservice.s3.PostPolicyGenerator;
33 | import org.signal.storageservice.storage.GroupsManager;
34 | import org.signal.storageservice.storage.protos.groups.Group;
35 | import org.signal.storageservice.storage.protos.groups.GroupChange;
36 | import org.signal.storageservice.storage.protos.groups.GroupChangeResponse;
37 | import org.signal.storageservice.storage.protos.groups.GroupResponse;
38 | import org.signal.storageservice.storage.protos.groups.GroupChanges;
39 |
40 | @Path("/v1/groups")
41 | public class GroupsV1Controller extends GroupsController {
42 |
43 | public GroupsV1Controller(
44 | Clock clock,
45 | GroupsManager groupsManager,
46 | ServerSecretParams serverSecretParams,
47 | PolicySigner policySigner,
48 | PostPolicyGenerator policyGenerator,
49 | GroupConfiguration groupConfiguration,
50 | ExternalGroupCredentialGenerator externalGroupCredentialGenerator) {
51 | super(clock, groupsManager, serverSecretParams, policySigner, policyGenerator, groupConfiguration, externalGroupCredentialGenerator);
52 | }
53 |
54 | @Override
55 | @Timed
56 | @GET
57 | @Produces(ProtocolBufferMediaType.APPLICATION_PROTOBUF)
58 | public CompletableFuture getGroup(@Auth GroupUser user) {
59 | return super.getGroup(user)
60 | .thenApply(response -> {
61 | if (response.getEntity() instanceof final GroupResponse gr) {
62 | return Response.fromResponse(response).entity(gr.getGroup()).build();
63 | }
64 | return response;
65 | });
66 | }
67 |
68 | @Override
69 | @Timed
70 | @GET
71 | @Produces(ProtocolBufferMediaType.APPLICATION_PROTOBUF)
72 | @Path("/logs/{fromVersion}")
73 | public CompletableFuture getGroupLogs(
74 | @Auth GroupUser user,
75 | @HeaderParam(jakarta.ws.rs.core.HttpHeaders.USER_AGENT) String userAgent,
76 | @HeaderParam("Cached-Send-Endorsements") Long ignored_usedByV2Only,
77 | @PathParam("fromVersion") int fromVersion,
78 | @QueryParam("limit") @DefaultValue("64") int limit,
79 | @QueryParam("maxSupportedChangeEpoch") Optional maxSupportedChangeEpoch,
80 | @QueryParam("includeFirstState") boolean includeFirstState,
81 | @QueryParam("includeLastState") boolean includeLastState) {
82 | return super.getGroupLogs(user, userAgent, Instant.now().getEpochSecond(), fromVersion, limit, maxSupportedChangeEpoch, includeFirstState, includeLastState)
83 | .thenApply(response -> {
84 | if (response.getEntity() instanceof final GroupChanges gc) {
85 | return Response.fromResponse(response).entity(gc.toBuilder().clearGroupSendEndorsementsResponse().build()).build();
86 | }
87 | return response;
88 | });
89 | }
90 |
91 | @Override
92 | @Timed
93 | @PUT
94 | @Produces(ProtocolBufferMediaType.APPLICATION_PROTOBUF)
95 | @Consumes(ProtocolBufferMediaType.APPLICATION_PROTOBUF)
96 | public CompletableFuture createGroup(@Auth GroupUser user, @NoUnknownFields Group group) {
97 | return super.createGroup(user, group)
98 | .thenApply(response -> Response.fromResponse(response).entity(null).build());
99 | }
100 |
101 | @Override
102 | @Timed
103 | @PATCH
104 | @Produces(ProtocolBufferMediaType.APPLICATION_PROTOBUF)
105 | @Consumes(ProtocolBufferMediaType.APPLICATION_PROTOBUF)
106 | public CompletableFuture modifyGroup(
107 | @Auth GroupUser user,
108 | @HeaderParam(jakarta.ws.rs.core.HttpHeaders.USER_AGENT) String userAgent,
109 | @QueryParam("inviteLinkPassword") String inviteLinkPasswordString,
110 | @NoUnknownFields GroupChange.Actions submittedActions) {
111 | return super.modifyGroup(user, userAgent, inviteLinkPasswordString, submittedActions)
112 | .thenApply(response -> {
113 | if (response.getEntity() instanceof final GroupChangeResponse gcr) {
114 | return Response.fromResponse(response).entity(gcr.getGroupChange()).build();
115 | }
116 | return response;
117 | });
118 | }
119 |
120 | }
121 |
--------------------------------------------------------------------------------
/src/main/java/org/signal/storageservice/storage/GroupLogTable.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2020 Signal Messenger, LLC
3 | * SPDX-License-Identifier: AGPL-3.0-only
4 | */
5 |
6 | package org.signal.storageservice.storage;
7 |
8 | import static com.codahale.metrics.MetricRegistry.name;
9 |
10 | import com.codahale.metrics.MetricRegistry;
11 | import com.codahale.metrics.SharedMetricRegistries;
12 | import com.codahale.metrics.Timer;
13 | import com.google.api.gax.rpc.ResponseObserver;
14 | import com.google.api.gax.rpc.StreamController;
15 | import com.google.cloud.bigtable.data.v2.BigtableDataClient;
16 | import com.google.cloud.bigtable.data.v2.models.Mutation;
17 | import com.google.cloud.bigtable.data.v2.models.Query;
18 | import com.google.cloud.bigtable.data.v2.models.Row;
19 | import com.google.protobuf.ByteString;
20 | import com.google.protobuf.InvalidProtocolBufferException;
21 | import java.util.LinkedList;
22 | import java.util.List;
23 | import java.util.concurrent.CompletableFuture;
24 | import javax.annotation.Nullable;
25 | import org.signal.storageservice.metrics.StorageMetrics;
26 | import org.signal.storageservice.storage.protos.groups.Group;
27 | import org.signal.storageservice.storage.protos.groups.GroupChange;
28 | import org.signal.storageservice.storage.protos.groups.GroupChanges.GroupChangeState;
29 | import org.signal.storageservice.util.Conversions;
30 | import org.signal.storageservice.util.Pair;
31 |
32 | public class GroupLogTable extends Table {
33 |
34 | public static final String FAMILY = "l";
35 |
36 | public static final String COLUMN_VERSION = "v";
37 | public static final String COLUMN_CHANGE = "c";
38 | public static final String COLUMN_STATE = "s";
39 |
40 | private final MetricRegistry metricRegistry = SharedMetricRegistries.getOrCreate(StorageMetrics.NAME);
41 | private final Timer appendTimer = metricRegistry.timer(name(GroupLogTable.class, "append" ));
42 | private final Timer getFromVersionTimer = metricRegistry.timer(name(GroupLogTable.class, "getFromVersion"));
43 |
44 | public GroupLogTable(BigtableDataClient client, String tableId) {
45 | super(client, tableId);
46 | }
47 |
48 | public CompletableFuture append(ByteString groupId, int version, GroupChange groupChange, Group group) {
49 | return setIfEmpty(appendTimer,
50 | getRowId(groupId, version),
51 | FAMILY, COLUMN_CHANGE,
52 | Mutation.create()
53 | .setCell(FAMILY, ByteString.copyFromUtf8(COLUMN_CHANGE), 0L, groupChange.toByteString())
54 | .setCell(FAMILY, COLUMN_VERSION, 0, String.valueOf(version))
55 | .setCell(FAMILY, ByteString.copyFromUtf8(COLUMN_STATE), 0L, group.toByteString()));
56 | }
57 |
58 | public CompletableFuture, Boolean>> getRecordsFromVersion(ByteString groupId,
59 | @Nullable Integer maxSupportedChangeEpoch, boolean includeFirstState, boolean includeLastState,
60 | int fromVersionInclusive, int toVersionExclusive, int currentVersion) {
61 |
62 | Timer.Context timerContext = getFromVersionTimer.time();
63 | CompletableFuture, Boolean>> future = new CompletableFuture<>();
64 | Query query = Query.create(tableId);
65 |
66 | query.range(getRowId(groupId, fromVersionInclusive), getRowId(groupId, toVersionExclusive));
67 |
68 | client.readRowsAsync(query, new ResponseObserver<>() {
69 | final List results = new LinkedList<>();
70 | boolean seenCurrentVersion = false;
71 |
72 | @Override
73 | public void onStart(StreamController controller) {}
74 |
75 | @Override
76 | public void onResponse(Row response) {
77 | try {
78 | GroupChange groupChange = GroupChange.parseFrom(response.getCells(FAMILY, COLUMN_CHANGE).stream().findFirst().orElseThrow().getValue());
79 | Group groupState = Group.parseFrom(response.getCells(FAMILY, COLUMN_STATE).stream().findFirst().orElseThrow().getValue());
80 | if (groupState.getVersion() == currentVersion) {
81 | seenCurrentVersion = true;
82 | }
83 | GroupChangeState.Builder groupChangeStateBuilder = GroupChangeState.newBuilder().setGroupChange(groupChange);
84 | if (maxSupportedChangeEpoch == null || maxSupportedChangeEpoch < groupChange.getChangeEpoch()
85 | || (includeFirstState && groupState.getVersion() == fromVersionInclusive)
86 | || (includeLastState && groupState.getVersion() == toVersionExclusive - 1)) {
87 | groupChangeStateBuilder.setGroupState(groupState);
88 | }
89 | results.add(groupChangeStateBuilder.build());
90 | } catch (InvalidProtocolBufferException e) {
91 | future.completeExceptionally(e);
92 | }
93 | }
94 |
95 | @Override
96 | public void onError(Throwable t) {
97 | timerContext.close();
98 | future.completeExceptionally(t);
99 | }
100 |
101 | @Override
102 | public void onComplete() {
103 | timerContext.close();
104 | future.complete(new Pair<>(results, seenCurrentVersion));
105 | }
106 | });
107 |
108 | return future;
109 | }
110 |
111 | private ByteString getRowId(ByteString groupId, int version) {
112 | return groupId.concat(ByteString.copyFromUtf8("#")).concat(ByteString.copyFrom(Conversions.intToByteArray(version)));
113 | }
114 | }
115 |
--------------------------------------------------------------------------------
/src/main/java/org/signal/storageservice/util/Conversions.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2020 Signal Messenger, LLC
3 | * SPDX-License-Identifier: AGPL-3.0-only
4 | */
5 |
6 | package org.signal.storageservice.util;
7 |
8 | public class Conversions {
9 |
10 | public static byte intsToByteHighAndLow(int highValue, int lowValue) {
11 | return (byte)((highValue << 4 | lowValue) & 0xFF);
12 | }
13 |
14 | public static int highBitsToInt(byte value) {
15 | return (value & 0xFF) >> 4;
16 | }
17 |
18 | public static int lowBitsToInt(byte value) {
19 | return (value & 0xF);
20 | }
21 |
22 | public static int highBitsToMedium(int value) {
23 | return (value >> 12);
24 | }
25 |
26 | public static int lowBitsToMedium(int value) {
27 | return (value & 0xFFF);
28 | }
29 |
30 | public static byte[] shortToByteArray(int value) {
31 | byte[] bytes = new byte[2];
32 | shortToByteArray(bytes, 0, value);
33 | return bytes;
34 | }
35 |
36 | public static int shortToByteArray(byte[] bytes, int offset, int value) {
37 | bytes[offset+1] = (byte)value;
38 | bytes[offset] = (byte)(value >> 8);
39 | return 2;
40 | }
41 |
42 | public static int shortToLittleEndianByteArray(byte[] bytes, int offset, int value) {
43 | bytes[offset] = (byte)value;
44 | bytes[offset+1] = (byte)(value >> 8);
45 | return 2;
46 | }
47 |
48 | public static byte[] mediumToByteArray(int value) {
49 | byte[] bytes = new byte[3];
50 | mediumToByteArray(bytes, 0, value);
51 | return bytes;
52 | }
53 |
54 | public static int mediumToByteArray(byte[] bytes, int offset, int value) {
55 | bytes[offset + 2] = (byte)value;
56 | bytes[offset + 1] = (byte)(value >> 8);
57 | bytes[offset] = (byte)(value >> 16);
58 | return 3;
59 | }
60 |
61 | public static byte[] intToByteArray(int value) {
62 | byte[] bytes = new byte[4];
63 | intToByteArray(bytes, 0, value);
64 | return bytes;
65 | }
66 |
67 | public static int intToByteArray(byte[] bytes, int offset, int value) {
68 | bytes[offset + 3] = (byte)value;
69 | bytes[offset + 2] = (byte)(value >> 8);
70 | bytes[offset + 1] = (byte)(value >> 16);
71 | bytes[offset] = (byte)(value >> 24);
72 | return 4;
73 | }
74 |
75 | public static int intToLittleEndianByteArray(byte[] bytes, int offset, int value) {
76 | bytes[offset] = (byte)value;
77 | bytes[offset+1] = (byte)(value >> 8);
78 | bytes[offset+2] = (byte)(value >> 16);
79 | bytes[offset+3] = (byte)(value >> 24);
80 | return 4;
81 | }
82 |
83 | public static byte[] longToByteArray(long l) {
84 | byte[] bytes = new byte[8];
85 | longToByteArray(bytes, 0, l);
86 | return bytes;
87 | }
88 |
89 | public static int longToByteArray(byte[] bytes, int offset, long value) {
90 | bytes[offset + 7] = (byte)value;
91 | bytes[offset + 6] = (byte)(value >> 8);
92 | bytes[offset + 5] = (byte)(value >> 16);
93 | bytes[offset + 4] = (byte)(value >> 24);
94 | bytes[offset + 3] = (byte)(value >> 32);
95 | bytes[offset + 2] = (byte)(value >> 40);
96 | bytes[offset + 1] = (byte)(value >> 48);
97 | bytes[offset] = (byte)(value >> 56);
98 | return 8;
99 | }
100 |
101 | public static int longTo4ByteArray(byte[] bytes, int offset, long value) {
102 | bytes[offset + 3] = (byte)value;
103 | bytes[offset + 2] = (byte)(value >> 8);
104 | bytes[offset + 1] = (byte)(value >> 16);
105 | bytes[offset + 0] = (byte)(value >> 24);
106 | return 4;
107 | }
108 |
109 | public static int byteArrayToShort(byte[] bytes) {
110 | return byteArrayToShort(bytes, 0);
111 | }
112 |
113 | public static int byteArrayToShort(byte[] bytes, int offset) {
114 | return
115 | (bytes[offset] & 0xff) << 8 | (bytes[offset + 1] & 0xff);
116 | }
117 |
118 | // The SSL patented 3-byte Value.
119 | public static int byteArrayToMedium(byte[] bytes, int offset) {
120 | return
121 | (bytes[offset] & 0xff) << 16 |
122 | (bytes[offset + 1] & 0xff) << 8 |
123 | (bytes[offset + 2] & 0xff);
124 | }
125 |
126 | public static int byteArrayToInt(byte[] bytes) {
127 | return byteArrayToInt(bytes, 0);
128 | }
129 |
130 | public static int byteArrayToInt(byte[] bytes, int offset) {
131 | return
132 | (bytes[offset] & 0xff) << 24 |
133 | (bytes[offset + 1] & 0xff) << 16 |
134 | (bytes[offset + 2] & 0xff) << 8 |
135 | (bytes[offset + 3] & 0xff);
136 | }
137 |
138 | public static int byteArrayToIntLittleEndian(byte[] bytes, int offset) {
139 | return
140 | (bytes[offset + 3] & 0xff) << 24 |
141 | (bytes[offset + 2] & 0xff) << 16 |
142 | (bytes[offset + 1] & 0xff) << 8 |
143 | (bytes[offset] & 0xff);
144 | }
145 |
146 | public static long byteArrayToLong(byte[] bytes) {
147 | return byteArrayToLong(bytes, 0);
148 | }
149 |
150 | public static long byteArray4ToLong(byte[] bytes, int offset) {
151 | return
152 | ((bytes[offset + 0] & 0xffL) << 24) |
153 | ((bytes[offset + 1] & 0xffL) << 16) |
154 | ((bytes[offset + 2] & 0xffL) << 8) |
155 | ((bytes[offset + 3] & 0xffL));
156 | }
157 |
158 | public static long byteArrayToLong(byte[] bytes, int offset) {
159 | return
160 | ((bytes[offset] & 0xffL) << 56) |
161 | ((bytes[offset + 1] & 0xffL) << 48) |
162 | ((bytes[offset + 2] & 0xffL) << 40) |
163 | ((bytes[offset + 3] & 0xffL) << 32) |
164 | ((bytes[offset + 4] & 0xffL) << 24) |
165 | ((bytes[offset + 5] & 0xffL) << 16) |
166 | ((bytes[offset + 6] & 0xffL) << 8) |
167 | ((bytes[offset + 7] & 0xffL));
168 | }
169 | }
170 |
--------------------------------------------------------------------------------
/src/main/java/org/signal/storageservice/storage/StorageManifestsTable.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2020 Signal Messenger, LLC
3 | * SPDX-License-Identifier: AGPL-3.0-only
4 | */
5 |
6 | package org.signal.storageservice.storage;
7 |
8 | import com.codahale.metrics.MetricRegistry;
9 | import com.codahale.metrics.SharedMetricRegistries;
10 | import com.codahale.metrics.Timer;
11 | import com.google.cloud.bigtable.data.v2.BigtableDataClient;
12 | import com.google.cloud.bigtable.data.v2.models.Filters;
13 | import com.google.cloud.bigtable.data.v2.models.Mutation;
14 | import com.google.cloud.bigtable.data.v2.models.Row;
15 | import com.google.cloud.bigtable.data.v2.models.RowCell;
16 | import com.google.cloud.bigtable.data.v2.models.RowMutation;
17 | import com.google.protobuf.ByteString;
18 | import org.signal.storageservice.auth.User;
19 | import org.signal.storageservice.metrics.StorageMetrics;
20 | import org.signal.storageservice.storage.protos.contacts.StorageManifest;
21 |
22 | import java.util.List;
23 | import java.util.Optional;
24 | import java.util.concurrent.CompletableFuture;
25 |
26 | import static com.codahale.metrics.MetricRegistry.name;
27 |
28 | public class StorageManifestsTable extends Table {
29 |
30 | static final String FAMILY = "m";
31 |
32 | static final String COLUMN_VERSION = "ver";
33 | static final String COLUMN_DATA = "dat";
34 |
35 | private final MetricRegistry metricRegistry = SharedMetricRegistries.getOrCreate(StorageMetrics.NAME);
36 | private final Timer getTimer = metricRegistry.timer(name(StorageManifestsTable.class, "get" ));
37 | private final Timer setTimer = metricRegistry.timer(name(StorageManifestsTable.class, "create" ));
38 | private final Timer getIfNotVersionTimer = metricRegistry.timer(name(StorageManifestsTable.class, "getIfNotVersion"));
39 | private final Timer deleteManifestTimer = metricRegistry.timer(name(StorageManifestsTable.class, "delete" ));
40 |
41 | public StorageManifestsTable(BigtableDataClient client, String tableId) {
42 | super(client, tableId);
43 | }
44 |
45 | /**
46 | * Updates the {@link StorageManifest} for the given user. The update is applied if and only if no manifest exists for
47 | * the given user or the given {@code manifest}'s version is exactly one greater than the version of the
48 | * existing manifest.
49 | *
50 | * @param user the user for whom to store an updated manifest
51 | * @param manifest the updated manifest to store
52 | *
53 | * @return a future that yields {@code true} if the manifest was updated or {@code false} otherwise
54 | */
55 | public CompletableFuture set(User user, StorageManifest manifest) {
56 | Mutation updateManifestMutation = Mutation.create()
57 | .setCell(FAMILY, COLUMN_VERSION, 0, String.valueOf(manifest.getVersion()))
58 | .setCell(FAMILY, ByteString.copyFromUtf8(COLUMN_DATA), 0, manifest.getValue());
59 |
60 | return setIfValueOrEmpty(setTimer, getRowKeyForManifest(user), FAMILY, COLUMN_VERSION, String.valueOf(manifest.getVersion() - 1), updateManifestMutation);
61 | }
62 |
63 | public CompletableFuture> get(User user) {
64 | return toFuture(client.readRowAsync(tableId, getRowKeyForManifest(user)), getTimer).thenApply(this::getManifestFromRow);
65 | }
66 |
67 | public CompletableFuture> getIfNotVersion(User user, long version) {
68 | return toFuture(client.readRowAsync(tableId, getRowKeyForManifest(user), Filters.FILTERS.condition(Filters.FILTERS.chain()
69 | .filter(Filters.FILTERS.key().exactMatch(getRowKeyForManifest(user)))
70 | .filter(Filters.FILTERS.family().exactMatch(FAMILY))
71 | .filter(Filters.FILTERS.qualifier().exactMatch(COLUMN_VERSION))
72 | .filter(Filters.FILTERS.value().exactMatch(String.valueOf(version))))
73 | .then(Filters.FILTERS.block())
74 | .otherwise(Filters.FILTERS.pass())),
75 | getIfNotVersionTimer).thenApply(this::getManifestFromRow);
76 | }
77 |
78 | public CompletableFuture clear(final User user) {
79 | return toFuture(client.mutateRowAsync(RowMutation.create(tableId, getRowKeyForManifest(user)).deleteRow()), deleteManifestTimer);
80 | }
81 |
82 | private ByteString getRowKeyForManifest(User user) {
83 | return ByteString.copyFromUtf8(user.getUuid().toString() + "#manifest");
84 | }
85 |
86 | private Optional getManifestFromRow(Row row) {
87 | if (row == null) return Optional.empty();
88 |
89 | StorageManifest.Builder contactsManifest = StorageManifest.newBuilder();
90 | List manifestCells = row.getCells(FAMILY);
91 |
92 | contactsManifest.setVersion(Long.valueOf(manifestCells.stream().filter(cell -> COLUMN_VERSION.equals(cell.getQualifier().toStringUtf8())).findFirst().orElseThrow().getValue().toStringUtf8()));
93 | contactsManifest.setValue(manifestCells.stream().filter(cell -> COLUMN_DATA.equals(cell.getQualifier().toStringUtf8())).findFirst().orElseThrow().getValue());
94 |
95 | return Optional.of(contactsManifest.build());
96 | }
97 |
98 | }
99 |
--------------------------------------------------------------------------------
/src/main/java/org/signal/storageservice/storage/Table.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2020 Signal Messenger, LLC
3 | * SPDX-License-Identifier: AGPL-3.0-only
4 | */
5 |
6 | package org.signal.storageservice.storage;
7 |
8 | import com.codahale.metrics.Timer;
9 | import com.google.api.core.ApiFuture;
10 | import com.google.api.core.ApiFutureCallback;
11 | import com.google.api.core.ApiFutures;
12 | import com.google.cloud.bigtable.data.v2.BigtableDataClient;
13 | import com.google.cloud.bigtable.data.v2.models.ConditionalRowMutation;
14 | import com.google.cloud.bigtable.data.v2.models.Mutation;
15 | import com.google.cloud.bigtable.data.v2.models.TableId;
16 | import com.google.common.util.concurrent.MoreExecutors;
17 | import com.google.protobuf.ByteString;
18 |
19 | import java.util.concurrent.CompletableFuture;
20 |
21 | import static com.google.cloud.bigtable.data.v2.models.Filters.FILTERS;
22 |
23 | abstract class Table {
24 |
25 | final BigtableDataClient client;
26 | final TableId tableId;
27 |
28 | public Table(final BigtableDataClient client, final String tableId) {
29 | this.client = client;
30 | this.tableId = TableId.of(tableId);
31 | }
32 |
33 | /**
34 | * Applies a mutation to the given row if and only if the cell with the given column family/name has exactly the given
35 | * value or the identified cell is empty.
36 | *
37 | * @param timer a timer to measure the duration of the operation
38 | * @param rowId the ID of the row to potentially mutate
39 | * @param columnFamily the column family of the cell to check for a specific value
40 | * @param columnName the column name of the cell to check for a specific value
41 | * @param columnEquals the value for which to check in the identified cell
42 | * @param mutation the mutation to apply if {@code columnEquals} exactly matches the existing value in the identified
43 | * cell or if the identified cell is empty
44 | *
45 | * @return a future that yields {@code true} if the identified row was modified or {@code false} otherwise
46 | * */
47 | CompletableFuture setIfValueOrEmpty(Timer timer, ByteString rowId, String columnFamily, String columnName, String columnEquals, Mutation mutation) {
48 | return setIfValue(timer, rowId, columnFamily, columnName, columnEquals, mutation)
49 | .thenCompose(mutated -> mutated
50 | ? CompletableFuture.completedFuture(true)
51 | : setIfEmpty(timer, rowId, columnFamily, columnName, mutation));
52 | }
53 |
54 | /**
55 | * Applies a mutation to the given row if and only if the cell with the given column family/name has exactly the given
56 | * value.
57 | *
58 | * @param timer a timer to measure the duration of the operation
59 | * @param rowId the ID of the row to potentially mutate
60 | * @param columnFamily the column family of the cell to check for a specific value
61 | * @param columnName the column name of the cell to check for a specific value
62 | * @param columnEquals the value for which to check in the identified cell
63 | * @param mutation the mutation to apply if {@code columnEquals} exactly matches the existing value in the identified
64 | * cell
65 | *
66 | * @return a future that yields {@code true} if the identified row was modified or {@code false} otherwise
67 | */
68 | CompletableFuture setIfValue(Timer timer, ByteString rowId, String columnFamily, String columnName, String columnEquals, Mutation mutation) {
69 | return toFuture(client.checkAndMutateRowAsync(ConditionalRowMutation.create(tableId, rowId)
70 | .condition(FILTERS.chain()
71 | .filter(FILTERS.family().exactMatch(columnFamily))
72 | .filter(FILTERS.qualifier().exactMatch(columnName))
73 | .filter(FILTERS.value().exactMatch(columnEquals)))
74 | .then(mutation)), timer);
75 | }
76 |
77 | /**
78 | * Applies a mutation to the given row if and only if the cell with the given column family/name is empty.
79 | *
80 | * @param timer a timer to measure the duration of the operation
81 | * @param rowId the ID of the row to potentially mutate
82 | * @param columnFamily the column family of the cell to check for a specific value
83 | * @param columnName the column name of the cell to check for a specific value
84 | * @param mutation the mutation to apply if the identified cell is empty
85 | *
86 | * @return a future that yields {@code true} if the identified row was modified or {@code false} otherwise
87 | */
88 | CompletableFuture setIfEmpty(Timer timer, ByteString rowId, String columnFamily, String columnName, Mutation mutation) {
89 | return toFuture(client.checkAndMutateRowAsync(ConditionalRowMutation.create(tableId, rowId)
90 | .condition(FILTERS.chain()
91 | .filter(FILTERS.family().exactMatch(columnFamily))
92 | .filter(FILTERS.qualifier().exactMatch(columnName))
93 | // See https://github.com/google/re2/wiki/Syntax; `\C` is "any byte", and so this matches any
94 | // non-empty value. Note that the mutation is applied in an `otherwise` clause.
95 | .filter(FILTERS.value().regex("\\C+")))
96 | .otherwise(mutation)), timer)
97 | // Note that we apply the mutation only if the predicate does NOT match, and so we invert `predicateMatched` to
98 | // indicate that we have (or haven't) mutated the row
99 | .thenApply(predicateMatched -> !predicateMatched);
100 | }
101 |
102 |
103 | static CompletableFuture toFuture(ApiFuture future, Timer timer) {
104 | Timer.Context timerContext = timer.time();
105 | CompletableFuture result = new CompletableFuture<>();
106 |
107 | ApiFutures.addCallback(future, new ApiFutureCallback() {
108 | @Override
109 | public void onFailure(Throwable t) {
110 | timerContext.close();
111 | result.completeExceptionally(t);
112 | }
113 |
114 | @Override
115 | public void onSuccess(T t) {
116 | timerContext.close();
117 | result.complete(t);
118 | }
119 | }, MoreExecutors.directExecutor());
120 |
121 | return result;
122 | }
123 |
124 | }
125 |
--------------------------------------------------------------------------------
/src/main/java/org/signal/storageservice/providers/ProtocolBufferMessageBodyProvider.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright © 2019 Smoke Turner, LLC (github@smoketurner.com)
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package org.signal.storageservice.providers;
17 |
18 | import com.google.protobuf.InvalidProtocolBufferException;
19 | import com.google.protobuf.Message;
20 | import com.google.protobuf.TextFormat;
21 | import com.google.protobuf.util.JsonFormat;
22 |
23 | import jakarta.ws.rs.Consumes;
24 | import jakarta.ws.rs.Produces;
25 | import jakarta.ws.rs.WebApplicationException;
26 | import jakarta.ws.rs.core.MediaType;
27 | import jakarta.ws.rs.core.MultivaluedMap;
28 | import jakarta.ws.rs.ext.MessageBodyReader;
29 | import jakarta.ws.rs.ext.MessageBodyWriter;
30 | import jakarta.ws.rs.ext.Provider;
31 | import java.io.IOException;
32 | import java.io.InputStream;
33 | import java.io.InputStreamReader;
34 | import java.io.OutputStream;
35 | import java.lang.annotation.Annotation;
36 | import java.lang.reflect.Method;
37 | import java.lang.reflect.Type;
38 | import java.nio.charset.StandardCharsets;
39 | import java.util.Map;
40 | import java.util.concurrent.ConcurrentHashMap;
41 |
42 | /**
43 | * A Jersey provider which enables using Protocol Buffers to parse request entities into objects and
44 | * generate response entities from objects.
45 | */
46 | @Provider
47 | @Consumes({
48 | ProtocolBufferMediaType.APPLICATION_PROTOBUF,
49 | ProtocolBufferMediaType.APPLICATION_PROTOBUF_TEXT,
50 | ProtocolBufferMediaType.APPLICATION_PROTOBUF_JSON
51 | })
52 | @Produces({
53 | ProtocolBufferMediaType.APPLICATION_PROTOBUF,
54 | ProtocolBufferMediaType.APPLICATION_PROTOBUF_TEXT,
55 | ProtocolBufferMediaType.APPLICATION_PROTOBUF_JSON
56 | })
57 | public class ProtocolBufferMessageBodyProvider
58 | implements MessageBodyReader, MessageBodyWriter {
59 |
60 | private final Map, Method> methodCache = new ConcurrentHashMap<>();
61 |
62 | @Override
63 | public boolean isReadable(
64 | final Class type,
65 | final Type genericType,
66 | final Annotation[] annotations,
67 | final MediaType mediaType) {
68 | return Message.class.isAssignableFrom(type);
69 | }
70 |
71 | @Override
72 | public Message readFrom(
73 | final Class type,
74 | final Type genericType,
75 | final Annotation[] annotations,
76 | final MediaType mediaType,
77 | final MultivaluedMap httpHeaders,
78 | final InputStream entityStream)
79 | throws IOException {
80 |
81 | final Method newBuilder =
82 | methodCache.computeIfAbsent(
83 | type,
84 | t -> {
85 | try {
86 | return t.getMethod("newBuilder");
87 | } catch (Exception e) {
88 | return null;
89 | }
90 | });
91 |
92 | final Message.Builder builder;
93 | try {
94 | builder = (Message.Builder) newBuilder.invoke(type);
95 | } catch (Exception e) {
96 | throw new WebApplicationException(e);
97 | }
98 |
99 | if (mediaType.getSubtype().contains("text-format")) {
100 | TextFormat.merge(new InputStreamReader(entityStream, StandardCharsets.UTF_8), builder);
101 | return builder.build();
102 | } else if (mediaType.getSubtype().contains("json-format")) {
103 | JsonFormat.parser()
104 | .merge(new InputStreamReader(entityStream, StandardCharsets.UTF_8), builder);
105 | return builder.build();
106 | } else {
107 | return builder.mergeFrom(entityStream).build();
108 | }
109 | }
110 |
111 | @Override
112 | public long getSize(
113 | final Message m,
114 | final Class type,
115 | final Type genericType,
116 | final Annotation[] annotations,
117 | final MediaType mediaType) {
118 |
119 | if (mediaType.getSubtype().contains("text-format")) {
120 | final String formatted = TextFormat.printer().escapingNonAscii(false).printToString(m);
121 | return formatted.getBytes(StandardCharsets.UTF_8).length;
122 | } else if (mediaType.getSubtype().contains("json-format")) {
123 | try {
124 | final String formatted = JsonFormat.printer().omittingInsignificantWhitespace().print(m);
125 | return formatted.getBytes(StandardCharsets.UTF_8).length;
126 | } catch (InvalidProtocolBufferException e) {
127 | // invalid protocol message
128 | return -1L;
129 | }
130 | }
131 |
132 | return m.getSerializedSize();
133 | }
134 |
135 | @Override
136 | public boolean isWriteable(
137 | final Class type,
138 | final Type genericType,
139 | final Annotation[] annotations,
140 | final MediaType mediaType) {
141 | return Message.class.isAssignableFrom(type);
142 | }
143 |
144 | @Override
145 | public void writeTo(
146 | final Message m,
147 | final Class type,
148 | final Type genericType,
149 | final Annotation[] annotations,
150 | final MediaType mediaType,
151 | final MultivaluedMap httpHeaders,
152 | final OutputStream entityStream)
153 | throws IOException {
154 |
155 | if (mediaType.getSubtype().contains("text-format")) {
156 | entityStream.write(m.toString().getBytes(StandardCharsets.UTF_8));
157 | } else if (mediaType.getSubtype().contains("json-format")) {
158 | final String formatted = JsonFormat.printer().omittingInsignificantWhitespace().print(m);
159 | entityStream.write(formatted.getBytes(StandardCharsets.UTF_8));
160 | } else {
161 | m.writeTo(entityStream);
162 | }
163 | }
164 | }
165 |
--------------------------------------------------------------------------------
/src/main/java/org/signal/storageservice/metrics/MetricsHttpChannelListener.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2024 Signal Messenger, LLC
3 | * SPDX-License-Identifier: AGPL-3.0-only
4 | */
5 |
6 | package org.signal.storageservice.metrics;
7 |
8 | import com.google.common.annotations.VisibleForTesting;
9 | import com.google.common.net.HttpHeaders;
10 | import io.dropwizard.core.setup.Environment;
11 | import io.micrometer.core.instrument.MeterRegistry;
12 | import io.micrometer.core.instrument.Metrics;
13 | import io.micrometer.core.instrument.Tags;
14 | import jakarta.ws.rs.container.ContainerRequestContext;
15 | import jakarta.ws.rs.container.ContainerResponseContext;
16 | import jakarta.ws.rs.container.ContainerResponseFilter;
17 | import java.io.IOException;
18 | import java.util.Optional;
19 | import javax.annotation.Nullable;
20 | import org.eclipse.jetty.server.Connector;
21 | import org.eclipse.jetty.server.HttpChannel;
22 | import org.eclipse.jetty.server.Request;
23 | import org.eclipse.jetty.util.component.Container;
24 | import org.eclipse.jetty.util.component.LifeCycle;
25 | import org.glassfish.jersey.server.ExtendedUriInfo;
26 | import org.signal.storageservice.util.UriInfoUtil;
27 | import org.slf4j.Logger;
28 | import org.slf4j.LoggerFactory;
29 |
30 | /**
31 | * Gathers and reports HTTP request metrics at the Jetty container level, which sits above Jersey. In order to get
32 | * templated Jersey request paths, it implements {@link ContainerResponseFilter}, in order to give itself access to the
33 | * template.
34 | *
35 | * It implements {@link LifeCycle.Listener} without overriding methods, so that it can be an event listener that
36 | * Dropwizard will attach to the container—the {@link Container.Listener} implementation is where it attaches
37 | * itself to any {@link Connector}s.
38 | */
39 | public class MetricsHttpChannelListener implements HttpChannel.Listener, Container.Listener, LifeCycle.Listener,
40 | ContainerResponseFilter {
41 |
42 | private static final Logger logger = LoggerFactory.getLogger(MetricsHttpChannelListener.class);
43 |
44 | private record RequestInfo(String path, String method, int statusCode, @Nullable String userAgent) {
45 | }
46 |
47 | // Use the same counter namespace as the now-retired MetricsRequestEventListener for continuity
48 | @VisibleForTesting
49 | static final String REQUEST_COUNTER_NAME =
50 | "org.signal.storageservice.metrics.MetricsRequestEventListener.request";
51 |
52 | @VisibleForTesting
53 | static final String REQUEST_BYTES_COUNTER_NAME =
54 | MetricsUtil.name(MetricsHttpChannelListener.class, "requestBytes");
55 |
56 | @VisibleForTesting
57 | static final String RESPONSE_BYTES_COUNTER_NAME =
58 | MetricsUtil.name(MetricsHttpChannelListener.class, "responseBytes");
59 |
60 | @VisibleForTesting
61 | static final String URI_INFO_PROPERTY_NAME = MetricsHttpChannelListener.class.getName() + ".uriInfo";
62 |
63 | @VisibleForTesting
64 | static final String PATH_TAG = "path";
65 |
66 | @VisibleForTesting
67 | static final String METHOD_TAG = "method";
68 |
69 | @VisibleForTesting
70 | static final String STATUS_CODE_TAG = "status";
71 |
72 | private final MeterRegistry meterRegistry;
73 |
74 | public MetricsHttpChannelListener() {
75 | this(Metrics.globalRegistry);
76 | }
77 |
78 | @VisibleForTesting
79 | MetricsHttpChannelListener(final MeterRegistry meterRegistry) {
80 | this.meterRegistry = meterRegistry;
81 | }
82 |
83 | public void configure(final Environment environment) {
84 | // register as ContainerResponseFilter
85 | environment.jersey().register(this);
86 |
87 | // hook into lifecycle events, to react to the Connector being added
88 | environment.lifecycle().addEventListener(this);
89 | }
90 |
91 | @Override
92 | public void onRequestFailure(final Request request, final Throwable failure) {
93 |
94 | if (logger.isDebugEnabled()) {
95 | final RequestInfo requestInfo = getRequestInfo(request);
96 |
97 | logger.debug("Request failure: {} {} ({}) [{}] ",
98 | requestInfo.method(),
99 | requestInfo.path(),
100 | requestInfo.userAgent(),
101 | requestInfo.statusCode(), failure);
102 | }
103 | }
104 |
105 | @Override
106 | public void onResponseFailure(Request request, Throwable failure) {
107 |
108 | if (failure instanceof org.eclipse.jetty.io.EofException) {
109 | // the client disconnected early
110 | return;
111 | }
112 |
113 | final RequestInfo requestInfo = getRequestInfo(request);
114 |
115 | logger.warn("Response failure: {} {} ({}) [{}] ",
116 | requestInfo.method(),
117 | requestInfo.path(),
118 | requestInfo.userAgent(),
119 | requestInfo.statusCode(), failure);
120 | }
121 |
122 | @Override
123 | public void onComplete(final Request request) {
124 |
125 | final RequestInfo requestInfo = getRequestInfo(request);
126 |
127 | final Tags tags = Tags.of(
128 | PATH_TAG, requestInfo.path(),
129 | METHOD_TAG, requestInfo.method(),
130 | STATUS_CODE_TAG, String.valueOf(requestInfo.statusCode()))
131 | .and(UserAgentTagUtil.getPlatformTag(requestInfo.userAgent()));
132 |
133 | meterRegistry.counter(REQUEST_COUNTER_NAME, tags).increment();
134 | meterRegistry.counter(REQUEST_BYTES_COUNTER_NAME, tags).increment(request.getContentRead());
135 | meterRegistry.counter(RESPONSE_BYTES_COUNTER_NAME, tags).increment(request.getResponse().getContentCount());
136 | }
137 |
138 | @Override
139 | public void beanAdded(final Container parent, final Object child) {
140 | if (child instanceof Connector connector) {
141 | connector.addBean(this);
142 | }
143 | }
144 |
145 | @Override
146 | public void beanRemoved(final Container parent, final Object child) {
147 | }
148 |
149 | @Override
150 | public void filter(final ContainerRequestContext requestContext, final ContainerResponseContext responseContext)
151 | throws IOException {
152 | requestContext.setProperty(URI_INFO_PROPERTY_NAME, requestContext.getUriInfo());
153 | }
154 |
155 | private RequestInfo getRequestInfo(Request request) {
156 | final String path = Optional.ofNullable(request.getAttribute(URI_INFO_PROPERTY_NAME))
157 | .map(attr -> UriInfoUtil.getPathTemplate((ExtendedUriInfo) attr))
158 | .orElseGet(() -> Optional.ofNullable(request.getPathInfo()).orElse("unknown"));
159 |
160 | final String method = Optional.ofNullable(request.getMethod()).orElse("unknown");
161 |
162 | // Response cannot be null, but its status might not always reflect an actual response status, since it gets
163 | // initialized to 200
164 | final int status = request.getResponse().getStatus();
165 |
166 | @Nullable final String userAgent = request.getHeader(HttpHeaders.USER_AGENT);
167 |
168 | return new RequestInfo(path, method, status, userAgent);
169 | }
170 |
171 | }
172 |
--------------------------------------------------------------------------------
/src/main/java/org/signal/storageservice/storage/StorageItemsTable.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2020 Signal Messenger, LLC
3 | * SPDX-License-Identifier: AGPL-3.0-only
4 | */
5 |
6 | package org.signal.storageservice.storage;
7 |
8 | import static com.codahale.metrics.MetricRegistry.name;
9 |
10 | import com.codahale.metrics.MetricRegistry;
11 | import com.codahale.metrics.SharedMetricRegistries;
12 | import com.codahale.metrics.Timer;
13 | import com.google.api.gax.rpc.ResponseObserver;
14 | import com.google.api.gax.rpc.StreamController;
15 | import com.google.cloud.bigtable.data.v2.BigtableDataClient;
16 | import com.google.cloud.bigtable.data.v2.models.BulkMutation;
17 | import com.google.cloud.bigtable.data.v2.models.Mutation;
18 | import com.google.cloud.bigtable.data.v2.models.Query;
19 | import com.google.cloud.bigtable.data.v2.models.Row;
20 | import com.google.protobuf.ByteString;
21 | import java.util.ArrayList;
22 | import java.util.LinkedList;
23 | import java.util.List;
24 | import java.util.concurrent.CompletableFuture;
25 | import org.apache.commons.codec.binary.Hex;
26 | import org.signal.storageservice.auth.User;
27 | import org.signal.storageservice.metrics.StorageMetrics;
28 | import org.signal.storageservice.storage.protos.contacts.StorageItem;
29 |
30 | public class StorageItemsTable extends Table {
31 |
32 | public static final String FAMILY = "c";
33 | public static final String ROW_KEY = "contact";
34 |
35 | public static final String COLUMN_DATA = "d";
36 | public static final String COLUMN_KEY = "k";
37 |
38 | public static final int MAX_MUTATIONS = 100_000;
39 | public static final int MUTATIONS_PER_INSERT = 2;
40 |
41 | private final MetricRegistry metricRegistry = SharedMetricRegistries.getOrCreate(StorageMetrics.NAME);
42 | private final Timer getTimer = metricRegistry.timer(name(StorageItemsTable.class, "get"));
43 | private final Timer setTimer = metricRegistry.timer(name(StorageItemsTable.class, "create"));
44 | private final Timer getKeysToDeleteTimer = metricRegistry.timer(name(StorageItemsTable.class, "getKeysToDelete"));
45 | private final Timer deleteKeysTimer = metricRegistry.timer(name(StorageItemsTable.class, "deleteKeys"));
46 |
47 | public StorageItemsTable(BigtableDataClient client, String tableId) {
48 | super(client, tableId);
49 | }
50 |
51 | public CompletableFuture set(final User user, final List inserts, final List deletes) {
52 | final List bulkMutations = new ArrayList<>();
53 | bulkMutations.add(BulkMutation.create(tableId));
54 |
55 | int mutations = 0;
56 |
57 | for (final StorageItem insert : inserts) {
58 | if (mutations + 2 > MAX_MUTATIONS) {
59 | bulkMutations.add(BulkMutation.create(tableId));
60 | mutations = 0;
61 | }
62 |
63 | bulkMutations.getLast().add(getRowKeyFor(user, insert.getKey()),
64 | Mutation.create()
65 | // each setCell() counts as mutation. If the below code changes, update MUTATIONS_PER_INSERT
66 | .setCell(FAMILY, ByteString.copyFromUtf8(COLUMN_DATA), 0, insert.getValue())
67 | .setCell(FAMILY, ByteString.copyFromUtf8(COLUMN_KEY), 0, insert.getKey()));
68 |
69 | mutations += 2;
70 | }
71 |
72 | for (ByteString delete : deletes) {
73 | if (mutations == MAX_MUTATIONS) {
74 | bulkMutations.add(BulkMutation.create(tableId));
75 | mutations = 0;
76 | }
77 |
78 | bulkMutations.getLast().add(getRowKeyFor(user, delete), Mutation.create().deleteRow());
79 |
80 | mutations += 1;
81 | }
82 |
83 | CompletableFuture future = CompletableFuture.completedFuture(null);
84 |
85 | for (final BulkMutation bulkMutation : bulkMutations) {
86 | future = future.thenCompose(ignored -> toFuture(client.bulkMutateRowsAsync(bulkMutation), setTimer));
87 | }
88 |
89 | return future;
90 | }
91 |
92 | public CompletableFuture clear(User user) {
93 | final Query query = Query.create(tableId);
94 | query.prefix(getRowKeyPrefixFor(user));
95 | query.limit(MAX_MUTATIONS);
96 |
97 | final CompletableFuture fetchRowsFuture = new CompletableFuture<>();
98 |
99 | final Timer.Context getKeysToDeleteTimerContext = getKeysToDeleteTimer.time();
100 | fetchRowsFuture.whenComplete((result, throwable) -> getKeysToDeleteTimerContext.close());
101 |
102 | client.readRowsAsync(query, new ResponseObserver<>() {
103 | private final BulkMutation bulkMutation = BulkMutation.create(tableId);
104 |
105 | @Override
106 | public void onStart(final StreamController streamController) {
107 | }
108 |
109 | @Override
110 | public void onResponse(final Row row) {
111 | bulkMutation.add(row.getKey(), Mutation.create().deleteRow());
112 | }
113 |
114 | @Override
115 | public void onError(final Throwable throwable) {
116 | fetchRowsFuture.completeExceptionally(throwable);
117 | }
118 |
119 | @Override
120 | public void onComplete() {
121 | fetchRowsFuture.complete(bulkMutation);
122 | }
123 | });
124 |
125 | return fetchRowsFuture.thenCompose(bulkMutation -> bulkMutation.getEntryCount() == 0
126 | ? CompletableFuture.completedFuture(null)
127 | : toFuture(client.bulkMutateRowsAsync(bulkMutation), deleteKeysTimer).thenCompose(ignored -> clear(user)));
128 | }
129 |
130 | public CompletableFuture> get(User user, List keys) {
131 | if (keys.isEmpty()) {
132 | throw new IllegalArgumentException("No keys");
133 | }
134 |
135 | Timer.Context timerContext = getTimer.time();
136 | CompletableFuture> future = new CompletableFuture<>();
137 | List results = new LinkedList<>();
138 | Query query = Query.create(tableId);
139 |
140 | for (ByteString key : keys) {
141 | query.rowKey(getRowKeyFor(user, key));
142 | }
143 |
144 | client.readRowsAsync(query, new ResponseObserver<>() {
145 | @Override
146 | public void onStart(StreamController controller) {
147 | }
148 |
149 | @Override
150 | public void onResponse(Row row) {
151 | ByteString key = row.getCells().stream().filter(cell -> COLUMN_KEY.equals(cell.getQualifier().toStringUtf8()))
152 | .findFirst().orElseThrow().getValue();
153 | ByteString value = row.getCells().stream()
154 | .filter(cell -> COLUMN_DATA.equals(cell.getQualifier().toStringUtf8())).findFirst().orElseThrow()
155 | .getValue();
156 |
157 | results.add(StorageItem.newBuilder()
158 | .setKey(key)
159 | .setValue(value)
160 | .build());
161 | }
162 |
163 | @Override
164 | public void onError(Throwable t) {
165 | timerContext.close();
166 | future.completeExceptionally(t);
167 | }
168 |
169 | @Override
170 | public void onComplete() {
171 | timerContext.close();
172 | future.complete(results);
173 | }
174 | });
175 |
176 | return future;
177 | }
178 |
179 | private ByteString getRowKeyFor(User user, ByteString key) {
180 | return ByteString.copyFromUtf8(
181 | user.getUuid().toString() + "#" + ROW_KEY + "#" + Hex.encodeHexString(key.toByteArray()));
182 | }
183 |
184 | private ByteString getRowKeyPrefixFor(User user) {
185 | return ByteString.copyFromUtf8(user.getUuid().toString() + "#" + ROW_KEY + "#");
186 | }
187 |
188 | }
189 |
--------------------------------------------------------------------------------
/src/main/java/org/signal/storageservice/controllers/StorageController.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2020 Signal Messenger, LLC
3 | * SPDX-License-Identifier: AGPL-3.0-only
4 | */
5 |
6 | package org.signal.storageservice.controllers;
7 |
8 | import static com.codahale.metrics.MetricRegistry.name;
9 |
10 | import com.codahale.metrics.annotation.Timed;
11 | import com.google.common.annotations.VisibleForTesting;
12 | import io.dropwizard.auth.Auth;
13 | import io.micrometer.core.instrument.DistributionSummary;
14 | import io.micrometer.core.instrument.Metrics;
15 | import io.micrometer.core.instrument.Tags;
16 | import java.time.Duration;
17 | import java.util.concurrent.CompletableFuture;
18 | import jakarta.ws.rs.Consumes;
19 | import jakarta.ws.rs.DELETE;
20 | import jakarta.ws.rs.GET;
21 | import jakarta.ws.rs.HeaderParam;
22 | import jakarta.ws.rs.PUT;
23 | import jakarta.ws.rs.Path;
24 | import jakarta.ws.rs.PathParam;
25 | import jakarta.ws.rs.Produces;
26 | import jakarta.ws.rs.WebApplicationException;
27 | import jakarta.ws.rs.core.HttpHeaders;
28 | import jakarta.ws.rs.core.Response;
29 | import jakarta.ws.rs.core.Response.Status;
30 | import org.signal.storageservice.auth.User;
31 | import org.signal.storageservice.metrics.UserAgentTagUtil;
32 | import org.signal.storageservice.providers.ProtocolBufferMediaType;
33 | import org.signal.storageservice.storage.StorageItemsTable;
34 | import org.signal.storageservice.storage.StorageManager;
35 | import org.signal.storageservice.storage.protos.contacts.ReadOperation;
36 | import org.signal.storageservice.storage.protos.contacts.StorageItems;
37 | import org.signal.storageservice.storage.protos.contacts.StorageManifest;
38 | import org.signal.storageservice.storage.protos.contacts.WriteOperation;
39 |
40 | @Path("/v1/storage")
41 | public class StorageController {
42 |
43 | private final StorageManager storageManager;
44 |
45 | @VisibleForTesting
46 | static final int MAX_READ_KEYS = 5120;
47 | // https://cloud.google.com/bigtable/quotas#limits-operations
48 |
49 | @VisibleForTesting
50 | static final int MAX_BULK_MUTATION_PAGES = 10;
51 |
52 | private static final String INSERT_DISTRIBUTION_SUMMARY_NAME = name(StorageController.class, "inserts");
53 | private static final String DELETE_DISTRIBUTION_SUMMARY_NAME = name(StorageController.class, "deletes");
54 | private static final String MUTATION_DISTRIBUTION_SUMMARY_NAME = name(StorageController.class, "mutations");
55 | private static final String READ_DISTRIBUTION_SUMMARY_NAME = name(StorageController.class, "reads");
56 | private static final String WRITE_REQUEST_SIZE_DISTRIBUTION_SUMMARY_NAME = name(StorageController.class, "writeRequestBytes");
57 |
58 | private static final String CLEAR_ALL_REQUEST_COUNTER_NAME = name(StorageController.class, "writeRequestClearAll");
59 |
60 | public StorageController(StorageManager storageManager) {
61 | this.storageManager = storageManager;
62 | }
63 |
64 | @Timed
65 | @GET
66 | @Path("/manifest")
67 | @Produces(ProtocolBufferMediaType.APPLICATION_PROTOBUF)
68 | public CompletableFuture getManifest(@Auth User user) {
69 | return storageManager.getManifest(user)
70 | .thenApply(manifest -> manifest.orElseThrow(() -> new WebApplicationException(Response.Status.NOT_FOUND)));
71 | }
72 |
73 | @Timed
74 | @GET
75 | @Path("/manifest/version/{version}")
76 | @Produces(ProtocolBufferMediaType.APPLICATION_PROTOBUF)
77 | public CompletableFuture getManifest(@Auth User user, @PathParam("version") long version) {
78 | return storageManager.getManifestIfNotVersion(user, version)
79 | .thenApply(manifest -> manifest.orElseThrow(() -> new WebApplicationException(Response.Status.NO_CONTENT)));
80 | }
81 |
82 |
83 | @Timed
84 | @PUT
85 | @Consumes(ProtocolBufferMediaType.APPLICATION_PROTOBUF)
86 | @Produces(ProtocolBufferMediaType.APPLICATION_PROTOBUF)
87 | public CompletableFuture write(@Auth User user, @HeaderParam(HttpHeaders.USER_AGENT) String userAgent, WriteOperation writeOperation) {
88 | if (!writeOperation.hasManifest()) {
89 | return CompletableFuture.failedFuture(new WebApplicationException(Response.Status.BAD_REQUEST));
90 | }
91 |
92 | distributionSummary(INSERT_DISTRIBUTION_SUMMARY_NAME, userAgent).record(writeOperation.getInsertItemCount());
93 | distributionSummary(DELETE_DISTRIBUTION_SUMMARY_NAME, userAgent).record(writeOperation.getDeleteKeyCount());
94 | distributionSummary(WRITE_REQUEST_SIZE_DISTRIBUTION_SUMMARY_NAME, userAgent).record(writeOperation.getSerializedSize());
95 |
96 | if (writeOperation.getClearAll()) {
97 | Metrics.counter(CLEAR_ALL_REQUEST_COUNTER_NAME, Tags.of(UserAgentTagUtil.getPlatformTag(userAgent))).increment();
98 | }
99 |
100 | final int mutations =
101 | writeOperation.getInsertItemCount() * StorageItemsTable.MUTATIONS_PER_INSERT + writeOperation.getDeleteKeyCount();
102 |
103 | distributionSummary(MUTATION_DISTRIBUTION_SUMMARY_NAME, userAgent).record(mutations);
104 |
105 | if (mutations > StorageItemsTable.MAX_MUTATIONS * MAX_BULK_MUTATION_PAGES) {
106 | return CompletableFuture.failedFuture(new WebApplicationException(Status.REQUEST_ENTITY_TOO_LARGE));
107 | }
108 |
109 | final CompletableFuture clearAllFuture = writeOperation.getClearAll()
110 | ? storageManager.clearItems(user)
111 | : CompletableFuture.completedFuture(null);
112 |
113 | return clearAllFuture.thenCompose(ignored -> storageManager.set(user, writeOperation.getManifest(), writeOperation.getInsertItemList(), writeOperation.getDeleteKeyList()))
114 | .thenApply(manifest -> {
115 | if (manifest.isPresent())
116 | return Response.status(409).entity(manifest.get()).build();
117 | else return Response.status(200).build();
118 | });
119 | }
120 |
121 | private static DistributionSummary distributionSummary(final String name, final String userAgent) {
122 | return DistributionSummary.builder(name)
123 | .publishPercentiles(0.75, 0.95, 0.99, 0.999)
124 | .distributionStatisticExpiry(Duration.ofMinutes(5))
125 | .tags(Tags.of(UserAgentTagUtil.getPlatformTag(userAgent)))
126 | .register(Metrics.globalRegistry);
127 | }
128 |
129 | @Timed
130 | @PUT
131 | @Path("/read")
132 | @Consumes(ProtocolBufferMediaType.APPLICATION_PROTOBUF)
133 | @Produces(ProtocolBufferMediaType.APPLICATION_PROTOBUF)
134 | public CompletableFuture read(@Auth User user, @HeaderParam(HttpHeaders.USER_AGENT) String userAgent, ReadOperation readOperation) {
135 | if (readOperation.getReadKeyList().isEmpty()) {
136 | return CompletableFuture.failedFuture(new WebApplicationException(Response.Status.BAD_REQUEST));
137 | }
138 |
139 | distributionSummary(READ_DISTRIBUTION_SUMMARY_NAME, userAgent).record(readOperation.getReadKeyCount());
140 |
141 | if (readOperation.getReadKeyCount() > MAX_READ_KEYS) {
142 | return CompletableFuture.failedFuture(new WebApplicationException(Status.REQUEST_ENTITY_TOO_LARGE));
143 | }
144 |
145 | return storageManager.getItems(user, readOperation.getReadKeyList())
146 | .thenApply(items -> StorageItems.newBuilder().addAllContacts(items).build());
147 | }
148 |
149 | @Timed
150 | @DELETE
151 | public CompletableFuture delete(@Auth User user) {
152 | return storageManager.delete(user).thenApply(v -> Response.status(Response.Status.OK).build());
153 | }
154 | }
155 |
--------------------------------------------------------------------------------
/src/main/java/org/signal/storageservice/StorageService.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2020-2021 Signal Messenger, LLC
3 | * SPDX-License-Identifier: AGPL-3.0-only
4 | */
5 |
6 | package org.signal.storageservice;
7 |
8 | import com.fasterxml.jackson.annotation.JsonAutoDetect;
9 | import com.fasterxml.jackson.annotation.PropertyAccessor;
10 | import com.fasterxml.jackson.databind.DeserializationFeature;
11 | import com.google.cloud.bigtable.data.v2.BigtableDataClient;
12 | import com.google.cloud.bigtable.data.v2.BigtableDataSettings;
13 | import com.google.common.collect.ImmutableMap;
14 | import com.google.common.collect.ImmutableSet;
15 | import io.dropwizard.auth.AuthFilter;
16 | import io.dropwizard.auth.PolymorphicAuthDynamicFeature;
17 | import io.dropwizard.auth.PolymorphicAuthValueFactoryProvider;
18 | import io.dropwizard.auth.basic.BasicCredentialAuthFilter;
19 | import io.dropwizard.auth.basic.BasicCredentials;
20 | import io.dropwizard.core.Application;
21 | import io.dropwizard.core.setup.Bootstrap;
22 | import io.dropwizard.core.setup.Environment;
23 | import java.time.Clock;
24 | import java.util.Set;
25 | import org.signal.libsignal.zkgroup.ServerSecretParams;
26 | import org.signal.libsignal.zkgroup.auth.ServerZkAuthOperations;
27 | import org.signal.storageservice.auth.ExternalGroupCredentialGenerator;
28 | import org.signal.storageservice.auth.ExternalServiceCredentialValidator;
29 | import org.signal.storageservice.auth.GroupUser;
30 | import org.signal.storageservice.auth.GroupUserAuthenticator;
31 | import org.signal.storageservice.auth.User;
32 | import org.signal.storageservice.auth.UserAuthenticator;
33 | import org.signal.storageservice.controllers.GroupsController;
34 | import org.signal.storageservice.controllers.GroupsV1Controller;
35 | import org.signal.storageservice.controllers.HealthCheckController;
36 | import org.signal.storageservice.controllers.ReadinessController;
37 | import org.signal.storageservice.controllers.StorageController;
38 | import org.signal.storageservice.filters.TimestampResponseFilter;
39 | import org.signal.storageservice.metrics.MetricsHttpChannelListener;
40 | import org.signal.storageservice.metrics.MetricsUtil;
41 | import org.signal.storageservice.providers.CompletionExceptionMapper;
42 | import org.signal.storageservice.providers.InvalidProtocolBufferExceptionMapper;
43 | import org.signal.storageservice.providers.ProtocolBufferMessageBodyProvider;
44 | import org.signal.storageservice.providers.ProtocolBufferValidationErrorMessageBodyWriter;
45 | import org.signal.storageservice.s3.PolicySigner;
46 | import org.signal.storageservice.s3.PostPolicyGenerator;
47 | import org.signal.storageservice.storage.GroupsManager;
48 | import org.signal.storageservice.storage.StorageManager;
49 | import org.signal.storageservice.util.UncaughtExceptionHandler;
50 | import org.signal.storageservice.util.logging.LoggingUnhandledExceptionMapper;
51 |
52 | public class StorageService extends Application {
53 |
54 | @Override
55 | public void initialize(Bootstrap bootstrap) { }
56 |
57 | @Override
58 | public void run(StorageServiceConfiguration config, Environment environment) throws Exception {
59 | MetricsUtil.configureRegistries(config, environment);
60 |
61 | UncaughtExceptionHandler.register();
62 |
63 | BigtableDataSettings bigtableDataSettings = BigtableDataSettings.newBuilder()
64 | .setProjectId(config.getBigTableConfiguration().getProjectId())
65 | .setInstanceId(config.getBigTableConfiguration().getInstanceId())
66 | .build();
67 | BigtableDataClient bigtableDataClient = BigtableDataClient.create(bigtableDataSettings);
68 | ServerSecretParams serverSecretParams = new ServerSecretParams(config.getZkConfiguration().getServerSecret());
69 | StorageManager storageManager = new StorageManager(bigtableDataClient, config.getBigTableConfiguration().getContactManifestsTableId(), config.getBigTableConfiguration().getContactsTableId());
70 | GroupsManager groupsManager = new GroupsManager(bigtableDataClient, config.getBigTableConfiguration().getGroupsTableId(), config.getBigTableConfiguration().getGroupLogsTableId());
71 |
72 | environment.getObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
73 | environment.getObjectMapper().setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE);
74 | environment.getObjectMapper().setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY);
75 |
76 | environment.jersey().register(ProtocolBufferMessageBodyProvider.class);
77 | environment.jersey().register(ProtocolBufferValidationErrorMessageBodyWriter.class);
78 | environment.jersey().register(InvalidProtocolBufferExceptionMapper.class);
79 | environment.jersey().register(CompletionExceptionMapper.class);
80 | environment.jersey().register(new LoggingUnhandledExceptionMapper());
81 |
82 | UserAuthenticator userAuthenticator = new UserAuthenticator(new ExternalServiceCredentialValidator(config.getAuthenticationConfiguration().getKey()));
83 | GroupUserAuthenticator groupUserAuthenticator = new GroupUserAuthenticator(new ServerZkAuthOperations(serverSecretParams));
84 | ExternalGroupCredentialGenerator externalGroupCredentialGenerator = new ExternalGroupCredentialGenerator(
85 | config.getGroupConfiguration().externalServiceSecret(), Clock.systemUTC());
86 |
87 | AuthFilter userAuthFilter = new BasicCredentialAuthFilter.Builder().setAuthenticator(userAuthenticator).buildAuthFilter();
88 | AuthFilter groupUserAuthFilter = new BasicCredentialAuthFilter.Builder().setAuthenticator(groupUserAuthenticator).buildAuthFilter();
89 |
90 | PolicySigner policySigner = new PolicySigner(config.getCdnConfiguration().getAccessSecret(), config.getCdnConfiguration().getRegion());
91 | PostPolicyGenerator postPolicyGenerator = new PostPolicyGenerator(config.getCdnConfiguration().getRegion(), config.getCdnConfiguration().getBucket(), config.getCdnConfiguration().getAccessKey());
92 |
93 | environment.jersey().register(new PolymorphicAuthDynamicFeature<>(ImmutableMap.of(User.class, userAuthFilter, GroupUser.class, groupUserAuthFilter)));
94 | environment.jersey().register(new PolymorphicAuthValueFactoryProvider.Binder<>(ImmutableSet.of(User.class, GroupUser.class)));
95 |
96 | environment.jersey().register(new TimestampResponseFilter(Clock.systemUTC()));
97 |
98 | environment.jersey().register(new HealthCheckController());
99 | environment.jersey().register(new ReadinessController(bigtableDataClient,
100 | Set.of(config.getBigTableConfiguration().getGroupsTableId(),
101 | config.getBigTableConfiguration().getGroupLogsTableId(),
102 | config.getBigTableConfiguration().getContactsTableId(),
103 | config.getBigTableConfiguration().getContactManifestsTableId()),
104 | config.getWarmUpConfiguration().count()));
105 | environment.jersey().register(new StorageController(storageManager));
106 | environment.jersey().register(new GroupsController(Clock.systemUTC(), groupsManager, serverSecretParams, policySigner, postPolicyGenerator, config.getGroupConfiguration(), externalGroupCredentialGenerator));
107 | environment.jersey().register(new GroupsV1Controller(Clock.systemUTC(), groupsManager, serverSecretParams, policySigner, postPolicyGenerator, config.getGroupConfiguration(), externalGroupCredentialGenerator));
108 |
109 | new MetricsHttpChannelListener().configure(environment);
110 |
111 | MetricsUtil.registerSystemResourceMetrics(environment);
112 | }
113 |
114 | public static void main(String[] argv) throws Exception {
115 | new StorageService().run(argv);
116 | }
117 | }
118 |
--------------------------------------------------------------------------------
/src/main/proto/Groups.proto:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2020 Signal Messenger, LLC
3 | * SPDX-License-Identifier: AGPL-3.0-only
4 | */
5 |
6 | syntax = "proto3";
7 |
8 | package signal;
9 |
10 | option java_package = "org.signal.storageservice.storage.protos.groups";
11 | option java_outer_classname = "GroupProtos";
12 | option java_multiple_files = true;
13 |
14 | message AvatarUploadAttributes {
15 | string key = 1;
16 | string credential = 2;
17 | string acl = 3;
18 | string algorithm = 4;
19 | string date = 5;
20 | string policy = 6;
21 | string signature = 7;
22 | }
23 |
24 | // Stored data
25 |
26 | message Member {
27 | enum Role {
28 | UNKNOWN = 0;
29 | DEFAULT = 1;
30 | ADMINISTRATOR = 2;
31 | }
32 |
33 | bytes userId = 1;
34 | Role role = 2;
35 | bytes profileKey = 3;
36 | bytes presentation = 4;
37 | uint32 joinedAtVersion = 5;
38 | }
39 |
40 | message MemberPendingProfileKey {
41 | Member member = 1;
42 | bytes addedByUserId = 2;
43 | uint64 timestamp = 3; // ms since epoch
44 | }
45 |
46 | message MemberPendingAdminApproval {
47 | bytes userId = 1;
48 | bytes profileKey = 2;
49 | bytes presentation = 3;
50 | uint64 timestamp = 4; // ms since epoch
51 | }
52 |
53 | message MemberBanned {
54 | bytes userId = 1;
55 | uint64 timestamp = 2; // ms since epoch
56 | }
57 |
58 | message AccessControl {
59 | enum AccessRequired {
60 | UNKNOWN = 0;
61 | ANY = 1;
62 | MEMBER = 2;
63 | ADMINISTRATOR = 3;
64 | UNSATISFIABLE = 4;
65 | }
66 |
67 | AccessRequired attributes = 1;
68 | AccessRequired members = 2;
69 | AccessRequired addFromInviteLink = 3;
70 | }
71 |
72 | message Group {
73 | bytes publicKey = 1;
74 | bytes title = 2;
75 | bytes description = 11;
76 | string avatar = 3;
77 | bytes disappearingMessagesTimer = 4;
78 | AccessControl accessControl = 5;
79 | uint32 version = 6;
80 | repeated Member members = 7;
81 | repeated MemberPendingProfileKey membersPendingProfileKey = 8;
82 | repeated MemberPendingAdminApproval membersPendingAdminApproval = 9;
83 | bytes inviteLinkPassword = 10;
84 | bool announcements_only = 12;
85 | repeated MemberBanned members_banned = 13;
86 | // next: 14
87 | }
88 |
89 | message GroupJoinInfo {
90 | bytes publicKey = 1;
91 | bytes title = 2;
92 | bytes description = 8;
93 | string avatar = 3;
94 | uint32 memberCount = 4;
95 | AccessControl.AccessRequired addFromInviteLink = 5;
96 | uint32 version = 6;
97 | bool pendingAdminApproval = 7;
98 | bool pendingAdminApprovalFull = 9;
99 | // next: 10
100 | }
101 |
102 | // Deltas
103 |
104 | message GroupChange {
105 |
106 | message Actions {
107 |
108 | message AddMemberAction {
109 | Member added = 1;
110 | bool joinFromInviteLink = 2;
111 | }
112 |
113 | message DeleteMemberAction {
114 | bytes deletedUserId = 1;
115 | }
116 |
117 | message ModifyMemberRoleAction {
118 | bytes userId = 1;
119 | Member.Role role = 2;
120 | }
121 |
122 | message ModifyMemberProfileKeyAction {
123 | bytes presentation = 1;
124 | bytes user_id = 2;
125 | bytes profile_key = 3;
126 | }
127 |
128 | message AddMemberPendingProfileKeyAction {
129 | MemberPendingProfileKey added = 1;
130 | }
131 |
132 | message DeleteMemberPendingProfileKeyAction {
133 | bytes deletedUserId = 1;
134 | }
135 |
136 | message PromoteMemberPendingProfileKeyAction {
137 | bytes presentation = 1;
138 | bytes user_id = 2;
139 | bytes profile_key = 3;
140 | }
141 |
142 | message PromoteMemberPendingPniAciProfileKeyAction {
143 | bytes presentation = 1;
144 | bytes user_id = 2;
145 | bytes pni = 3;
146 | bytes profile_key = 4;
147 | }
148 |
149 | message AddMemberPendingAdminApprovalAction {
150 | MemberPendingAdminApproval added = 1;
151 | }
152 |
153 | message DeleteMemberPendingAdminApprovalAction {
154 | bytes deletedUserId = 1;
155 | }
156 |
157 | message PromoteMemberPendingAdminApprovalAction {
158 | bytes userId = 1;
159 | Member.Role role = 2;
160 | }
161 |
162 | message AddMemberBannedAction {
163 | MemberBanned added = 1;
164 | }
165 |
166 | message DeleteMemberBannedAction {
167 | bytes deletedUserId = 1;
168 | }
169 |
170 | message ModifyTitleAction {
171 | bytes title = 1;
172 | }
173 |
174 | message ModifyDescriptionAction {
175 | bytes description = 1;
176 | }
177 |
178 | message ModifyAvatarAction {
179 | string avatar = 1;
180 | }
181 |
182 | message ModifyDisappearingMessageTimerAction {
183 | bytes timer = 1;
184 | }
185 |
186 | message ModifyAttributesAccessControlAction {
187 | AccessControl.AccessRequired attributesAccess = 1;
188 | }
189 |
190 | message ModifyMembersAccessControlAction {
191 | AccessControl.AccessRequired membersAccess = 1;
192 | }
193 |
194 | message ModifyAddFromInviteLinkAccessControlAction {
195 | AccessControl.AccessRequired addFromInviteLinkAccess = 1;
196 | }
197 |
198 | message ModifyInviteLinkPasswordAction {
199 | bytes inviteLinkPassword = 1;
200 | }
201 |
202 | message ModifyAnnouncementsOnlyAction {
203 | bool announcements_only = 1;
204 | }
205 |
206 | bytes sourceUuid = 1;
207 | // clients should not provide this value; the server will provide it in the response buffer to ensure the signature is binding to a particular group
208 | // if clients set it during a request the server will respond with 400.
209 | bytes group_id = 25;
210 | uint32 version = 2;
211 |
212 | repeated AddMemberAction addMembers = 3;
213 | repeated DeleteMemberAction deleteMembers = 4;
214 | repeated ModifyMemberRoleAction modifyMemberRoles = 5;
215 | repeated ModifyMemberProfileKeyAction modifyMemberProfileKeys = 6;
216 | repeated AddMemberPendingProfileKeyAction addMembersPendingProfileKey = 7;
217 | repeated DeleteMemberPendingProfileKeyAction deleteMembersPendingProfileKey = 8;
218 | repeated PromoteMemberPendingProfileKeyAction promoteMembersPendingProfileKey = 9;
219 | ModifyTitleAction modifyTitle = 10;
220 | ModifyAvatarAction modifyAvatar = 11;
221 | ModifyDisappearingMessageTimerAction modifyDisappearingMessageTimer = 12;
222 | ModifyAttributesAccessControlAction modifyAttributesAccess = 13;
223 | ModifyMembersAccessControlAction modifyMemberAccess = 14;
224 | ModifyAddFromInviteLinkAccessControlAction modifyAddFromInviteLinkAccess = 15; // change epoch = 1
225 | repeated AddMemberPendingAdminApprovalAction addMembersPendingAdminApproval = 16; // change epoch = 1
226 | repeated DeleteMemberPendingAdminApprovalAction deleteMembersPendingAdminApproval = 17; // change epoch = 1
227 | repeated PromoteMemberPendingAdminApprovalAction promoteMembersPendingAdminApproval = 18; // change epoch = 1
228 | ModifyInviteLinkPasswordAction modifyInviteLinkPassword = 19; // change epoch = 1
229 | ModifyDescriptionAction modifyDescription = 20; // change epoch = 2
230 | ModifyAnnouncementsOnlyAction modify_announcements_only = 21; // change epoch = 3
231 | repeated AddMemberBannedAction add_members_banned = 22; // change epoch = 4
232 | repeated DeleteMemberBannedAction delete_members_banned = 23; // change epoch = 4
233 | repeated PromoteMemberPendingPniAciProfileKeyAction promote_members_pending_pni_aci_profile_key = 24; // change epoch = 5
234 | // next: 26
235 | }
236 |
237 | bytes actions = 1;
238 | bytes serverSignature = 2;
239 | uint32 changeEpoch = 3;
240 | }
241 |
242 | // External credentials
243 |
244 | message ExternalGroupCredential {
245 | string token = 1;
246 | }
247 |
248 | // API responses
249 |
250 | message GroupResponse {
251 | Group group = 1;
252 | bytes group_send_endorsements_response = 2;
253 | }
254 |
255 | message GroupChanges {
256 | message GroupChangeState {
257 | GroupChange groupChange = 1;
258 | Group groupState = 2;
259 | }
260 |
261 | repeated GroupChangeState groupChanges = 1;
262 | bytes group_send_endorsements_response = 2;
263 | }
264 |
265 | message GroupChangeResponse {
266 | GroupChange group_change = 1;
267 | bytes group_send_endorsements_response = 2;
268 | }
269 |
--------------------------------------------------------------------------------
/mvnw.cmd:
--------------------------------------------------------------------------------
1 | @REM ----------------------------------------------------------------------------
2 | @REM Licensed to the Apache Software Foundation (ASF) under one
3 | @REM or more contributor license agreements. See the NOTICE file
4 | @REM distributed with this work for additional information
5 | @REM regarding copyright ownership. The ASF licenses this file
6 | @REM to you under the Apache License, Version 2.0 (the
7 | @REM "License"); you may not use this file except in compliance
8 | @REM with the License. You may obtain a copy of the License at
9 | @REM
10 | @REM http://www.apache.org/licenses/LICENSE-2.0
11 | @REM
12 | @REM Unless required by applicable law or agreed to in writing,
13 | @REM software distributed under the License is distributed on an
14 | @REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | @REM KIND, either express or implied. See the License for the
16 | @REM specific language governing permissions and limitations
17 | @REM under the License.
18 | @REM ----------------------------------------------------------------------------
19 |
20 | @REM ----------------------------------------------------------------------------
21 | @REM Apache Maven Wrapper startup batch script, version 3.2.0
22 | @REM
23 | @REM Required ENV vars:
24 | @REM JAVA_HOME - location of a JDK home dir
25 | @REM
26 | @REM Optional ENV vars
27 | @REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands
28 | @REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending
29 | @REM MAVEN_OPTS - parameters passed to the Java VM when running Maven
30 | @REM e.g. to debug Maven itself, use
31 | @REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
32 | @REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files
33 | @REM ----------------------------------------------------------------------------
34 |
35 | @REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on'
36 | @echo off
37 | @REM set title of command window
38 | title %0
39 | @REM enable echoing by setting MAVEN_BATCH_ECHO to 'on'
40 | @if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO%
41 |
42 | @REM set %HOME% to equivalent of $HOME
43 | if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%")
44 |
45 | @REM Execute a user defined script before this one
46 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre
47 | @REM check for pre script, once with legacy .bat ending and once with .cmd ending
48 | if exist "%USERPROFILE%\mavenrc_pre.bat" call "%USERPROFILE%\mavenrc_pre.bat" %*
49 | if exist "%USERPROFILE%\mavenrc_pre.cmd" call "%USERPROFILE%\mavenrc_pre.cmd" %*
50 | :skipRcPre
51 |
52 | @setlocal
53 |
54 | set ERROR_CODE=0
55 |
56 | @REM To isolate internal variables from possible post scripts, we use another setlocal
57 | @setlocal
58 |
59 | @REM ==== START VALIDATION ====
60 | if not "%JAVA_HOME%" == "" goto OkJHome
61 |
62 | echo.
63 | echo Error: JAVA_HOME not found in your environment. >&2
64 | echo Please set the JAVA_HOME variable in your environment to match the >&2
65 | echo location of your Java installation. >&2
66 | echo.
67 | goto error
68 |
69 | :OkJHome
70 | if exist "%JAVA_HOME%\bin\java.exe" goto init
71 |
72 | echo.
73 | echo Error: JAVA_HOME is set to an invalid directory. >&2
74 | echo JAVA_HOME = "%JAVA_HOME%" >&2
75 | echo Please set the JAVA_HOME variable in your environment to match the >&2
76 | echo location of your Java installation. >&2
77 | echo.
78 | goto error
79 |
80 | @REM ==== END VALIDATION ====
81 |
82 | :init
83 |
84 | @REM Find the project base dir, i.e. the directory that contains the folder ".mvn".
85 | @REM Fallback to current working directory if not found.
86 |
87 | set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR%
88 | IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir
89 |
90 | set EXEC_DIR=%CD%
91 | set WDIR=%EXEC_DIR%
92 | :findBaseDir
93 | IF EXIST "%WDIR%"\.mvn goto baseDirFound
94 | cd ..
95 | IF "%WDIR%"=="%CD%" goto baseDirNotFound
96 | set WDIR=%CD%
97 | goto findBaseDir
98 |
99 | :baseDirFound
100 | set MAVEN_PROJECTBASEDIR=%WDIR%
101 | cd "%EXEC_DIR%"
102 | goto endDetectBaseDir
103 |
104 | :baseDirNotFound
105 | set MAVEN_PROJECTBASEDIR=%EXEC_DIR%
106 | cd "%EXEC_DIR%"
107 |
108 | :endDetectBaseDir
109 |
110 | IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig
111 |
112 | @setlocal EnableExtensions EnableDelayedExpansion
113 | for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a
114 | @endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS%
115 |
116 | :endReadAdditionalConfig
117 |
118 | SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe"
119 | set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar"
120 | set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
121 |
122 | set WRAPPER_URL="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar"
123 |
124 | FOR /F "usebackq tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO (
125 | IF "%%A"=="wrapperUrl" SET WRAPPER_URL=%%B
126 | )
127 |
128 | @REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central
129 | @REM This allows using the maven wrapper in projects that prohibit checking in binary data.
130 | if exist %WRAPPER_JAR% (
131 | if "%MVNW_VERBOSE%" == "true" (
132 | echo Found %WRAPPER_JAR%
133 | )
134 | ) else (
135 | if not "%MVNW_REPOURL%" == "" (
136 | SET WRAPPER_URL="%MVNW_REPOURL%/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar"
137 | )
138 | if "%MVNW_VERBOSE%" == "true" (
139 | echo Couldn't find %WRAPPER_JAR%, downloading it ...
140 | echo Downloading from: %WRAPPER_URL%
141 | )
142 |
143 | powershell -Command "&{"^
144 | "$webclient = new-object System.Net.WebClient;"^
145 | "if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^
146 | "$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^
147 | "}"^
148 | "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%WRAPPER_URL%', '%WRAPPER_JAR%')"^
149 | "}"
150 | if "%MVNW_VERBOSE%" == "true" (
151 | echo Finished downloading %WRAPPER_JAR%
152 | )
153 | )
154 | @REM End of extension
155 |
156 | @REM If specified, validate the SHA-256 sum of the Maven wrapper jar file
157 | SET WRAPPER_SHA_256_SUM=""
158 | FOR /F "usebackq tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO (
159 | IF "%%A"=="wrapperSha256Sum" SET WRAPPER_SHA_256_SUM=%%B
160 | )
161 | IF NOT %WRAPPER_SHA_256_SUM%=="" (
162 | powershell -Command "&{"^
163 | "$hash = (Get-FileHash \"%WRAPPER_JAR%\" -Algorithm SHA256).Hash.ToLower();"^
164 | "If('%WRAPPER_SHA_256_SUM%' -ne $hash){"^
165 | " Write-Output 'Error: Failed to validate Maven wrapper SHA-256, your Maven wrapper might be compromised.';"^
166 | " Write-Output 'Investigate or delete %WRAPPER_JAR% to attempt a clean download.';"^
167 | " Write-Output 'If you updated your Maven version, you need to update the specified wrapperSha256Sum property.';"^
168 | " exit 1;"^
169 | "}"^
170 | "}"
171 | if ERRORLEVEL 1 goto error
172 | )
173 |
174 | @REM Provide a "standardized" way to retrieve the CLI args that will
175 | @REM work with both Windows and non-Windows executions.
176 | set MAVEN_CMD_LINE_ARGS=%*
177 |
178 | %MAVEN_JAVA_EXE% ^
179 | %JVM_CONFIG_MAVEN_PROPS% ^
180 | %MAVEN_OPTS% ^
181 | %MAVEN_DEBUG_OPTS% ^
182 | -classpath %WRAPPER_JAR% ^
183 | "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" ^
184 | %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %*
185 | if ERRORLEVEL 1 goto error
186 | goto end
187 |
188 | :error
189 | set ERROR_CODE=1
190 |
191 | :end
192 | @endlocal & set ERROR_CODE=%ERROR_CODE%
193 |
194 | if not "%MAVEN_SKIP_RC%"=="" goto skipRcPost
195 | @REM check for post script, once with legacy .bat ending and once with .cmd ending
196 | if exist "%USERPROFILE%\mavenrc_post.bat" call "%USERPROFILE%\mavenrc_post.bat"
197 | if exist "%USERPROFILE%\mavenrc_post.cmd" call "%USERPROFILE%\mavenrc_post.cmd"
198 | :skipRcPost
199 |
200 | @REM pause the script if MAVEN_BATCH_PAUSE is set to 'on'
201 | if "%MAVEN_BATCH_PAUSE%"=="on" pause
202 |
203 | if "%MAVEN_TERMINATE_CMD%"=="on" exit %ERROR_CODE%
204 |
205 | cmd /C exit /B %ERROR_CODE%
206 |
--------------------------------------------------------------------------------