Read delayed
36 | * message delivery for better understanding this feature.
37 | */
38 | @PublicEvolving
39 | public interface MessageDelayer This topic router is stateless and doesn't have any initialize logic. Make sure you don't
37 | * require some dynamic state.
38 | *
39 | * @param The PulsarSubscriber provides a unified interface for the Pulsar source to support all these
44 | * two types of subscribing mode.
45 | */
46 | @Internal
47 | public interface PulsarSubscriber extends Serializable {
48 |
49 | /**
50 | * Get a set of subscribed {@link TopicPartition}s. The method could throw {@link
51 | * IllegalStateException}, an extra try catch is required.
52 | *
53 | * @param generator The range for different partitions.
54 | * @param parallelism The parallelism of flink source.
55 | * @return A subscribed {@link TopicPartition} for each pulsar topic partition.
56 | */
57 | @SuppressWarnings("java:S112")
58 | Set If you implement this interface, make sure that each {@link TopicRange} would be assigned to a
38 | * specified source reader. Since flink parallelism is provided, make sure the pulsar message key's
39 | * hashcode is evenly distributed among these topic ranges.
40 | */
41 | @PublicEvolving
42 | @FunctionalInterface
43 | public interface RangeGenerator extends Serializable {
44 |
45 | /**
46 | * Generate range for the given topic.
47 | *
48 | * @param metadata The metadata of the topic.
49 | * @param parallelism The reader size for this topic.
50 | */
51 | List You just need to write a JUnit 5 test class and extends this suite class. All the helper
41 | * method list below would be ready.
42 | *
43 | * {@code PulsarSourceEnumeratorTest} would be a test example for how to use this base class. If
44 | * you have some setup logic, such as create topic or send message, just place then in a setup
45 | * method with annotation {@code @BeforeAll}. This setup method would not require {@code static}.
46 | *
47 | * @see PulsarRuntimeOperator for how to use the helper methods in this class.
48 | */
49 | @ExtendWith(TestLoggerExtension.class)
50 | @TestInstance(TestInstance.Lifecycle.PER_CLASS)
51 | public abstract class PulsarTestSuiteBase {
52 |
53 | @RegisterExtension
54 | final PulsarTestEnvironment environment =
55 | new PulsarTestEnvironment(runtime().withConfigs(runtimeConfigs()));
56 |
57 | /**
58 | * Choose the desired pulsar runtime as the test backend. The default test backend is a
59 | * singleton pulsar instance. Override this method when needed.
60 | */
61 | protected PulsarRuntime runtime() {
62 | return PulsarRuntime.singletonContainer();
63 | }
64 |
65 | /** Operate pulsar by acquiring a runtime operator. */
66 | protected PulsarRuntimeOperator operator() {
67 | return environment.operator();
68 | }
69 |
70 | /** Override the default runtime configuration. */
71 | protected MapString.hashCode(). */
39 | JAVA_HASH(
40 | "java-hash",
41 | text(
42 | "This hash would use %s to calculate the message key string's hash code.",
43 | code("String.hashCode()"))) {
44 | @Override
45 | public Hash getHash() {
46 | return JavaStringHash.getInstance();
47 | }
48 | },
49 | /**
50 | * Use Murmur3 hashing function. https://en.wikipedia.org/wiki/MurmurHash
52 | */
53 | MURMUR3_32_HASH(
54 | "murmur-3-32-hash",
55 | text(
56 | "This hash would calculate message key's hash code by using %s algorithm.",
57 | link("https://en.wikipedia.org/wiki/MurmurHash", "Murmur3"))) {
58 | @Override
59 | public Hash getHash() {
60 | return Murmur3Hash32.getInstance();
61 | }
62 | };
63 |
64 | private final String name;
65 | private final transient InlineElement desc;
66 |
67 | MessageKeyHash(String name, InlineElement desc) {
68 | this.name = name;
69 | this.desc = desc;
70 | }
71 |
72 | @Internal
73 | public abstract Hash getHash();
74 |
75 | @Override
76 | public String toString() {
77 | return name;
78 | }
79 |
80 | @Internal
81 | @Override
82 | public InlineElement getDescription() {
83 | return desc;
84 | }
85 | }
86 |
--------------------------------------------------------------------------------
/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/router/RoundRobinTopicRouter.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 |
19 | package org.apache.flink.connector.pulsar.sink.writer.router;
20 |
21 | import org.apache.flink.annotation.Internal;
22 | import org.apache.flink.connector.pulsar.sink.config.SinkConfiguration;
23 | import org.apache.flink.connector.pulsar.sink.writer.context.PulsarSinkContext;
24 | import org.apache.flink.connector.pulsar.source.enumerator.topic.TopicPartition;
25 | import org.apache.flink.util.Preconditions;
26 |
27 | import java.util.List;
28 | import java.util.concurrent.atomic.AtomicLong;
29 |
30 | import static org.apache.flink.util.Preconditions.checkArgument;
31 |
32 | /**
33 | * If you choose the {@link TopicRoutingMode#ROUND_ROBIN} policy, we would use this implementation.
34 | * We would pick the topic one by one in a fixed batch size.
35 | *
36 | * @param
39 | *
42 | *
43 | *
36 | *
39 | *
40 | *