bIter = b.iterator();
160 | while (aIter.hasNext() && bIter.hasNext()) {
161 | if (!headerEqualityByValue(aIter.next(), bIter.next()))
162 | return false;
163 | }
164 | return !aIter.hasNext() && !bIter.hasNext();
165 | }
166 |
167 | private boolean headerEqualityByValue(Header a, Header b) {
168 | // This is an alternative implementation of ConnectHeader::equals that use proper Value equality by value
169 | // (even if they are byte arrays)
170 | if (a == b) {
171 | return true;
172 | }
173 | if (!Objects.equals(a.key(), b.key())) {
174 | return false;
175 | }
176 | try {
177 | // This particular case is not handled by ConnectHeader::equals
178 | byte[] aBytes = a.value();
179 | byte[] bBytes = b.value();
180 | return Arrays.equals(aBytes, bBytes);
181 | } catch (ClassCastException e) {
182 | return a.value() == b.value();
183 | }
184 | }
185 |
186 |
187 | }
188 |
--------------------------------------------------------------------------------
/src/main/java/de/azapps/kafkabackup/common/record/RecordSerde.java:
--------------------------------------------------------------------------------
1 | package de.azapps.kafkabackup.common.record;
2 |
3 | import org.apache.kafka.common.header.Header;
4 | import org.apache.kafka.common.header.internals.RecordHeaders;
5 | import org.apache.kafka.common.record.TimestampType;
6 |
7 | import java.io.*;
8 | import java.nio.charset.StandardCharsets;
9 |
10 | /**
11 | * Record Format:
12 | * offset: int64
13 | * timestampType: int32 -2 if timestamp is null
14 | * [timestamp: int64] if timestampType != NO_TIMESTAMP_TYPE && timestamp != null
15 | * keyLength: int32
16 | * [key: byte[keyLength]] if keyLength >= 0
17 | * valueLength: int32
18 | * [value: byte[valueLength]] if valueLength >= 0
19 | * headerCount: int32
20 | * headers: Header[headerCount]
21 | *
22 | * Header Format:
23 | * headerKeyLength: int32
24 | * headerKey: byte[headerKeyLength]
25 | * headerValueLength: int32
26 | * [headerValue: byte[headerValueLength]] if headerValueLength >= 0
27 | */
28 | public class RecordSerde {
29 | public static Record read(String topic, int partition, InputStream inputStream) throws IOException {
30 | DataInputStream dataStream = new DataInputStream(inputStream);
31 | long offset = dataStream.readLong();
32 | int timestampTypeInt = dataStream.readInt();
33 | TimestampType timestampType;
34 | Long timestamp;
35 | // See comment in `write()`
36 | if (timestampTypeInt == -2) {
37 | timestampType = TimestampType.CREATE_TIME;
38 | timestamp=null;
39 | } else {
40 | switch (timestampTypeInt) {
41 | case -1:
42 | timestampType = TimestampType.NO_TIMESTAMP_TYPE;
43 | break;
44 | case 0:
45 | timestampType = TimestampType.CREATE_TIME;
46 | break;
47 | case 1:
48 | timestampType = TimestampType.LOG_APPEND_TIME;
49 | break;
50 | default:
51 | throw new RuntimeException("Unexpected TimestampType. Expected -1,0 or 1. Got " + timestampTypeInt);
52 | }
53 | if (timestampType != TimestampType.NO_TIMESTAMP_TYPE) {
54 | timestamp = dataStream.readLong();
55 | } else {
56 | timestamp = null;
57 | }
58 | }
59 | int keyLength = dataStream.readInt();
60 | byte[] key = null;
61 | if (keyLength >= 0) {
62 | key = new byte[keyLength];
63 | int readBytes = dataStream.read(key);
64 | if (readBytes != keyLength) {
65 | throw new IOException(String.format("Expected to read %d bytes, got %d", keyLength, readBytes));
66 | }
67 | }
68 |
69 | int valueLength = dataStream.readInt();
70 | byte[] value = null;
71 | if (valueLength >= 0) {
72 | value = new byte[valueLength];
73 | int readBytes = dataStream.read(value);
74 | if (readBytes != valueLength) {
75 | throw new IOException(String.format("Expected to read %d bytes, got %d", valueLength, readBytes));
76 | }
77 | }
78 | int headerCount = dataStream.readInt();
79 | RecordHeaders headers = new RecordHeaders();
80 | for (int i = 0; i < headerCount; i++) {
81 | // Key
82 | int headerKeyLength = dataStream.readInt();
83 | if (headerKeyLength < 0) {
84 | throw new RuntimeException("Invalid negative header key size " + headerKeyLength);
85 | }
86 | byte[] headerKeyBytes = new byte[headerKeyLength];
87 | int readBytes = dataStream.read(headerKeyBytes);
88 | if (readBytes != headerKeyLength) {
89 | throw new IOException(String.format("Expected to read %d bytes, got %d", headerKeyLength, readBytes));
90 | }
91 | String headerKey = new String(headerKeyBytes, StandardCharsets.UTF_8);
92 | // Value
93 | int headerValueLength = dataStream.readInt();
94 | byte[] headerValue = null;
95 | if (headerValueLength >= 0) {
96 | headerValue = new byte[headerValueLength];
97 | int hvReadBytes = dataStream.read(headerValue);
98 | if (hvReadBytes != headerValueLength) {
99 | throw new IOException(String.format("Expected to read %d bytes, got %d", headerValueLength, hvReadBytes));
100 | }
101 | }
102 | headers.add(headerKey, headerValue);
103 | }
104 |
105 | return new Record(topic, partition, key, value, offset, timestamp, timestampType, headers);
106 | }
107 |
108 | public static void write(OutputStream outputStream, Record record) throws IOException {
109 | DataOutputStream dataStream = new DataOutputStream(outputStream);
110 | dataStream.writeLong(record.kafkaOffset());
111 | // There is a special case where the timestamp type eqauls `CREATE_TIME` but is actually `null`.
112 | // This should not happen normally and I see it as a bug in the Client implementation of pykafka
113 | // But as Kafka accepts that value, so should Kafka Backup. Thus, this dirty workaround: we write the
114 | // timestamp type `-2` if the type is CREATE_TIME but the timestamp itself is null. Otherwise we would have
115 | // needed to change the byte format and for now I think this is the better solution.
116 | if (record.timestampType() == TimestampType.CREATE_TIME && record.timestamp() == null) {
117 | dataStream.writeInt(-2);
118 | } else {
119 | dataStream.writeInt(record.timestampType().id);
120 | if (record.timestampType() != TimestampType.NO_TIMESTAMP_TYPE) {
121 | dataStream.writeLong(record.timestamp());
122 | }
123 | }
124 | if (record.key() != null) {
125 | dataStream.writeInt(record.key().length);
126 | dataStream.write(record.key());
127 | } else {
128 | dataStream.writeInt(-1);
129 | }
130 | if (record.value() != null) {
131 | dataStream.writeInt(record.value().length);
132 | dataStream.write(record.value());
133 | } else {
134 | dataStream.writeInt(-1);
135 | }
136 | Header[] headers = record.headers().toArray();
137 | dataStream.writeInt(headers.length);
138 | for (Header header : record.headers()) {
139 | byte[] headerKeyBytes = header.key().getBytes(StandardCharsets.UTF_8);
140 | dataStream.writeInt(headerKeyBytes.length);
141 | dataStream.write(headerKeyBytes);
142 | if (header.value() != null) {
143 | dataStream.writeInt(header.value().length);
144 | dataStream.write(header.value());
145 | } else {
146 | dataStream.writeInt(-1);
147 | }
148 | }
149 | }
150 | }
151 |
--------------------------------------------------------------------------------
/src/main/java/de/azapps/kafkabackup/common/segment/SegmentIndex.java:
--------------------------------------------------------------------------------
1 | package de.azapps.kafkabackup.common.segment;
2 |
3 | import java.io.EOFException;
4 | import java.io.FileInputStream;
5 | import java.io.FileOutputStream;
6 | import java.io.IOException;
7 | import java.nio.file.Files;
8 | import java.nio.file.Path;
9 | import java.util.ArrayList;
10 | import java.util.List;
11 | import java.util.Optional;
12 |
13 | public class SegmentIndex {
14 | private static final byte V1_MAGIC_BYTE = 0x01;
15 | private Path indexFile;
16 | private List index = new ArrayList<>();
17 | private long lastValidRecordOffset = -1;
18 | private long lastValidIndexPosition = 1; // mind the magic byte!
19 | private FileOutputStream fileOutputStream;
20 | private FileInputStream fileInputStream;
21 |
22 | public SegmentIndex(Path indexFile) throws IOException, IndexException {
23 | this.indexFile = indexFile;
24 | initFile();
25 | while (true) {
26 | try {
27 | SegmentIndexEntry segmentIndexEntry = SegmentIndexEntry.fromStream(fileInputStream);
28 | if (segmentIndexEntry.getOffset() <= lastValidRecordOffset) {
29 | throw new IndexException("Offsets must be always increasing! There is something terribly wrong in your index!");
30 | }
31 | index.add(segmentIndexEntry);
32 | lastValidRecordOffset = segmentIndexEntry.getOffset();
33 | lastValidIndexPosition = fileInputStream.getChannel().position();
34 | } catch (EOFException e) {
35 | // reached End of File
36 | break;
37 | }
38 | }
39 | }
40 |
41 | private void initFile() throws IOException, IndexException {
42 | if (!Files.isRegularFile(indexFile)) {
43 | Files.createFile(indexFile);
44 | fileOutputStream = new FileOutputStream(indexFile.toFile());
45 | fileOutputStream.write(V1_MAGIC_BYTE);
46 | } else {
47 | fileOutputStream = new FileOutputStream(indexFile.toFile(), true);
48 | }
49 | this.fileInputStream = new FileInputStream(indexFile.toFile());
50 | byte[] v1Validation = new byte[1];
51 | if (fileInputStream.read(v1Validation) != 1 || v1Validation[0] != V1_MAGIC_BYTE) {
52 | throw new IndexException("Cannot validate Magic Byte in the beginning of the index " + indexFile);
53 | }
54 | }
55 |
56 | void addEntry(SegmentIndexEntry segmentIndexEntry) throws IOException, IndexException {
57 | if (segmentIndexEntry.getOffset() <= lastValidRecordOffset) {
58 | throw new IndexException("Offsets must be always increasing! There is something terribly wrong in your index!");
59 | }
60 | fileOutputStream.getChannel().position(lastValidIndexPosition);
61 | segmentIndexEntry.writeToStream(fileOutputStream);
62 | lastValidIndexPosition = fileOutputStream.getChannel().position();
63 | lastValidRecordOffset = segmentIndexEntry.getOffset();
64 | index.add(segmentIndexEntry);
65 | }
66 |
67 | Optional lastIndexEntry() {
68 | if (!index.isEmpty()) {
69 | return Optional.of(index.get(index.size() - 1));
70 | } else {
71 | return Optional.empty();
72 | }
73 | }
74 |
75 | long lastValidStartPosition() {
76 | if (!index.isEmpty()) {
77 | return index.get(index.size() - 1).recordFilePosition();
78 | } else {
79 | return 0L;
80 | }
81 |
82 | }
83 |
84 | Optional getByPosition(int position) {
85 | if (position >= index.size()) {
86 | return Optional.empty();
87 | } else {
88 | return Optional.of(index.get(position));
89 | }
90 | }
91 |
92 | Optional findByOffset(long offset) {
93 | for (SegmentIndexEntry current : index) {
94 | if (current.getOffset() == offset) {
95 | return Optional.of(current.recordFilePosition());
96 | }
97 | }
98 | return Optional.empty();
99 | }
100 |
101 | Optional findEarliestWithHigherOrEqualOffset(long offset) {
102 | for (SegmentIndexEntry current : index) {
103 | if (current.getOffset() >= offset) {
104 | return Optional.of(current.recordFilePosition());
105 | }
106 | }
107 | return Optional.empty();
108 | }
109 |
110 | int size() {
111 | return index.size();
112 | }
113 |
114 | public List index() {
115 | return index;
116 | }
117 |
118 | void flush() throws IOException {
119 | fileOutputStream.flush();
120 | }
121 |
122 | void close() throws IOException {
123 | fileInputStream.close();
124 | fileOutputStream.close();
125 | }
126 |
127 | public static class IndexException extends Exception {
128 | IndexException(String message) {
129 | super(message);
130 | }
131 | }
132 |
133 | }
134 |
--------------------------------------------------------------------------------
/src/main/java/de/azapps/kafkabackup/common/segment/SegmentIndexEntry.java:
--------------------------------------------------------------------------------
1 | package de.azapps.kafkabackup.common.segment;
2 |
3 | import java.io.*;
4 | import java.util.Objects;
5 |
6 | /**
7 | * Format:
8 | * offset: int64
9 | * recordFilePosition: int64
10 | * recordLength: int64
11 | */
12 | public class SegmentIndexEntry {
13 | private final long offset;
14 | private final long recordFilePosition;
15 | private final long recordByteLength;
16 |
17 | SegmentIndexEntry(long offset, long recordFilePosition, long recordByteLength) {
18 | this.offset = offset;
19 | this.recordFilePosition = recordFilePosition;
20 | this.recordByteLength = recordByteLength;
21 | }
22 |
23 | static SegmentIndexEntry fromStream(InputStream byteStream) throws IOException {
24 | DataInputStream stream = new DataInputStream(byteStream);
25 | long offset = stream.readLong();
26 | long recordFileOffset = stream.readLong();
27 | long recordByteLength = stream.readLong();
28 | return new SegmentIndexEntry(offset, recordFileOffset, recordByteLength);
29 | }
30 |
31 | public long getOffset() {
32 | return offset;
33 | }
34 |
35 | public long recordFilePosition() {
36 | return recordFilePosition;
37 | }
38 |
39 | public long recordByteLength() {
40 | return recordByteLength;
41 | }
42 |
43 | void writeToStream(OutputStream byteStream) throws IOException {
44 | DataOutputStream stream = new DataOutputStream(byteStream);
45 | stream.writeLong(offset);
46 | stream.writeLong(recordFilePosition);
47 | stream.writeLong(recordByteLength);
48 | }
49 |
50 | @Override
51 | public int hashCode() {
52 | return Objects.hash(offset, recordFilePosition, recordByteLength);
53 | }
54 |
55 | @Override
56 | public boolean equals(Object o) {
57 | if (this == o)
58 | return true;
59 | if (o == null || getClass() != o.getClass())
60 | return false;
61 |
62 | SegmentIndexEntry that = (SegmentIndexEntry) o;
63 |
64 | return Objects.equals(getOffset(), that.getOffset())
65 | && Objects.equals(recordFilePosition(), that.recordFilePosition())
66 | && Objects.equals(recordByteLength(), that.recordByteLength());
67 | }
68 |
69 | @Override
70 | public String toString() {
71 | return String.format("SegmentIndexEntry{offset: %d, recordFilePosition: %d, recordByteLength: %d}",
72 | offset, recordFilePosition, recordByteLength);
73 | }
74 | }
75 |
--------------------------------------------------------------------------------
/src/main/java/de/azapps/kafkabackup/common/segment/SegmentIndexRestore.java:
--------------------------------------------------------------------------------
1 | package de.azapps.kafkabackup.common.segment;
2 |
3 | import de.azapps.kafkabackup.common.record.Record;
4 |
5 | import java.io.EOFException;
6 | import java.io.IOException;
7 | import java.nio.file.Files;
8 | import java.nio.file.Path;
9 |
10 | public class SegmentIndexRestore {
11 | private final SegmentIndex segmentIndex;
12 | private final UnverifiedSegmentReader reader;
13 |
14 | public SegmentIndexRestore(Path segmentFile) throws IOException, RestoreException, SegmentIndex.IndexException {
15 | int partition = SegmentUtils.getPartitionFromSegment(segmentFile);
16 | long startOffset = SegmentUtils.getStartOffsetFromSegment(segmentFile);
17 | Path parent = segmentFile.toAbsolutePath().getParent();
18 | if (parent == null) {
19 | throw new RestoreException("Segment file " + segmentFile + " does not exist");
20 | }
21 | Path indexFile = SegmentUtils.indexFile(parent, partition, startOffset);
22 |
23 | if (!Files.isRegularFile(segmentFile)) {
24 | throw new RestoreException("Segment file " + segmentFile + " does not exist");
25 | }
26 | if (Files.isRegularFile(indexFile)) {
27 | throw new RestoreException("Index file " + indexFile + " must not exist");
28 | }
29 | segmentIndex = new SegmentIndex(indexFile);
30 | reader = new UnverifiedSegmentReader(segmentFile);
31 | }
32 |
33 | public void restore() throws IOException, SegmentIndex.IndexException {
34 | long lastPosition = 1; // mind the magic byte!
35 | while (true) {
36 | try {
37 | Record record = reader.read();
38 | long currentPosition = reader.position();
39 | SegmentIndexEntry indexEntry = new SegmentIndexEntry(record.kafkaOffset(), lastPosition, currentPosition - lastPosition);
40 | segmentIndex.addEntry(indexEntry);
41 | lastPosition = currentPosition;
42 | } catch (EOFException e) {
43 | break;
44 | }
45 | }
46 | segmentIndex.flush();
47 | segmentIndex.close();
48 | }
49 |
50 | public static class RestoreException extends Exception {
51 | RestoreException(String message) {
52 | super(message);
53 | }
54 | }
55 |
56 |
57 | }
58 |
--------------------------------------------------------------------------------
/src/main/java/de/azapps/kafkabackup/common/segment/SegmentReader.java:
--------------------------------------------------------------------------------
1 | package de.azapps.kafkabackup.common.segment;
2 |
3 | import de.azapps.kafkabackup.common.record.Record;
4 | import de.azapps.kafkabackup.common.record.RecordSerde;
5 |
6 | import java.io.EOFException;
7 | import java.io.FileInputStream;
8 | import java.io.IOException;
9 | import java.nio.channels.FileChannel;
10 | import java.nio.file.Files;
11 | import java.nio.file.Path;
12 | import java.util.ArrayList;
13 | import java.util.List;
14 | import java.util.Optional;
15 |
16 | public class SegmentReader {
17 | private final String topic;
18 | private final int partition;
19 | private final String filePrefix;
20 | private final SegmentIndex segmentIndex;
21 | private final FileInputStream recordInputStream;
22 | private final long lastValidStartPosition;
23 |
24 | public SegmentReader(String topic, int partition, Path topicDir, long startOffset) throws IOException, SegmentIndex.IndexException {
25 | this(topic, partition, topicDir, SegmentUtils.filePrefix(partition, startOffset));
26 | }
27 |
28 | public SegmentReader(String topic, int partition, Path topicDir, String filePrefix) throws IOException, SegmentIndex.IndexException {
29 | this.topic = topic;
30 | this.partition = partition;
31 | this.filePrefix = filePrefix;
32 |
33 | Path indexFile = SegmentUtils.indexFile(topicDir, filePrefix);
34 | Path recordFile = SegmentUtils.recordsFile(topicDir, filePrefix);
35 | if (!Files.isRegularFile(indexFile)) {
36 | throw new RuntimeException("Index for Segment not found: " + indexFile.toString());
37 | }
38 | if (!Files.isRegularFile(recordFile)) {
39 | throw new RuntimeException("Segment not found: " + recordFile.toString());
40 | }
41 | segmentIndex = new SegmentIndex(indexFile);
42 | recordInputStream = new FileInputStream(recordFile.toFile());
43 | SegmentUtils.ensureValidSegment(recordInputStream);
44 | lastValidStartPosition = segmentIndex.lastValidStartPosition();
45 | }
46 |
47 | public void seek(long offset) throws IOException {
48 | Optional optionalPosition = segmentIndex.findEarliestWithHigherOrEqualOffset(offset);
49 | if (optionalPosition.isPresent()) {
50 | recordInputStream.getChannel().position(optionalPosition.get());
51 | } else {
52 | // If we couldn't find such a record, skip to EOF. This will make sure that hasMoreData() returns false.
53 | FileChannel fileChannel = recordInputStream.getChannel();
54 | fileChannel.position(fileChannel.size());
55 | }
56 | }
57 |
58 | public boolean hasMoreData() throws IOException {
59 | return recordInputStream.getChannel().position() <= lastValidStartPosition;
60 | }
61 |
62 | public Record read() throws IOException {
63 | if (!hasMoreData()) {
64 | throw new EOFException("Already read the last valid record in topic " + topic + ", segment " + filePrefix);
65 | }
66 | return RecordSerde.read(topic, partition, recordInputStream);
67 | }
68 |
69 | public List readN(int n) throws IOException {
70 | List records = new ArrayList<>(n);
71 | while (hasMoreData() && records.size() < n) {
72 | Record record = read();
73 | records.add(record);
74 | }
75 | return records;
76 | }
77 |
78 | public List readFully() throws IOException {
79 | List records = new ArrayList<>(segmentIndex.size());
80 | while (hasMoreData()) {
81 | Record record = read();
82 | records.add(record);
83 | }
84 | return records;
85 | }
86 |
87 | public void close() throws IOException {
88 | recordInputStream.close();
89 | segmentIndex.close();
90 | }
91 | }
92 |
--------------------------------------------------------------------------------
/src/main/java/de/azapps/kafkabackup/common/segment/SegmentUtils.java:
--------------------------------------------------------------------------------
1 | package de.azapps.kafkabackup.common.segment;
2 |
3 | import java.io.FileInputStream;
4 | import java.io.IOException;
5 | import java.nio.file.Path;
6 | import java.nio.file.Paths;
7 | import java.util.regex.Matcher;
8 | import java.util.regex.Pattern;
9 |
10 | public class SegmentUtils {
11 |
12 | static final byte V1_MAGIC_BYTE = 0x01;
13 | private static final Pattern SEGMENT_PATTERN = Pattern.compile("^segment_partition_([0-9]+)_from_offset_([0-9]+)_records$");
14 |
15 | public static String filePrefix(int partition, long startOffset) {
16 | return String.format("segment_partition_%03d_from_offset_%010d", partition, startOffset);
17 | }
18 |
19 | static void ensureValidSegment(FileInputStream inputStream) throws IOException {
20 | inputStream.getChannel().position(0);
21 | byte[] v1Validation = new byte[1];
22 | if (inputStream.read(v1Validation) != 1 || v1Validation[0] != SegmentUtils.V1_MAGIC_BYTE) {
23 | throw new IOException("Cannot validate Magic Byte in the beginning of the Segment");
24 | }
25 | }
26 |
27 | public static Path indexFile(Path topicDir, int partition, long startOffset) {
28 | return indexFile(topicDir, filePrefix(partition, startOffset));
29 | }
30 |
31 | static Path indexFile(Path topicDir, String filePrefix) {
32 | return Paths.get(topicDir.toString(), filePrefix + "_index");
33 | }
34 |
35 | public static Path recordsFile(Path topicDir, int partition, long startOffset) {
36 | return recordsFile(topicDir, filePrefix(partition, startOffset));
37 | }
38 |
39 | static Path recordsFile(Path topicDir, String filePrefix) {
40 | return Paths.get(topicDir.toString(), filePrefix + "_records");
41 | }
42 |
43 | public static boolean isSegment(Path file) {
44 | Path fpath = file.getFileName();
45 | if (fpath == null) {
46 | return false;
47 | }
48 | Matcher m = SEGMENT_PATTERN.matcher(fpath.toString());
49 | return m.find();
50 | }
51 |
52 | public static int getPartitionFromSegment(Path file) {
53 | Path fpath = file.getFileName();
54 | if (fpath == null) {
55 | throw new RuntimeException("File " + file + " is not a Segment");
56 | }
57 | Matcher m = SEGMENT_PATTERN.matcher(fpath.toString());
58 | if (m.find()) {
59 | String partitionStr = m.group(1);
60 | return Integer.parseInt(partitionStr);
61 | } else {
62 | throw new RuntimeException("File " + file + " is not a Segment");
63 | }
64 | }
65 |
66 | public static long getStartOffsetFromSegment(Path file) {
67 | Path fpath = file.getFileName();
68 | if (fpath == null) {
69 | throw new RuntimeException("File " + file + " is not a Segment");
70 | }
71 | Matcher m = SEGMENT_PATTERN.matcher(fpath.toString());
72 | if (m.find()) {
73 | String offsetStr = m.group(2);
74 | return Long.parseLong(offsetStr);
75 | } else {
76 | throw new RuntimeException("File " + file + " is not a Segment");
77 | }
78 | }
79 |
80 | }
81 |
--------------------------------------------------------------------------------
/src/main/java/de/azapps/kafkabackup/common/segment/SegmentWriter.java:
--------------------------------------------------------------------------------
1 | package de.azapps.kafkabackup.common.segment;
2 |
3 | import de.azapps.kafkabackup.common.record.Record;
4 | import de.azapps.kafkabackup.common.record.RecordSerde;
5 |
6 | import java.io.FileInputStream;
7 | import java.io.FileOutputStream;
8 | import java.io.IOException;
9 | import java.nio.file.Files;
10 | import java.nio.file.Path;
11 | import java.util.Optional;
12 |
13 | public class SegmentWriter {
14 | private final String topic;
15 | private final int partition;
16 | private final long startOffset;
17 | private final SegmentIndex segmentIndex;
18 | private final FileOutputStream recordOutputStream;
19 |
20 | public SegmentWriter(String topic, int partition, long startOffset, Path topicDir) throws IOException, SegmentIndex.IndexException {
21 | this.topic = topic;
22 | this.partition = partition;
23 | this.startOffset = startOffset;
24 |
25 | Path indexFile = SegmentUtils.indexFile(topicDir, partition, startOffset);
26 | segmentIndex = new SegmentIndex(indexFile);
27 |
28 | Path recordFile = SegmentUtils.recordsFile(topicDir, partition, startOffset);
29 | if (!Files.isRegularFile(recordFile)) {
30 | Files.createFile(recordFile);
31 | recordOutputStream = new FileOutputStream(recordFile.toFile());
32 | recordOutputStream.write(SegmentUtils.V1_MAGIC_BYTE);
33 | } else {
34 | // Validate Magic Byte
35 | FileInputStream inputStream = new FileInputStream(recordFile.toFile());
36 | SegmentUtils.ensureValidSegment(inputStream);
37 | inputStream.close();
38 |
39 | // move to last committed position of the file
40 | recordOutputStream = new FileOutputStream(recordFile.toFile(), true);
41 | Optional optionalPreviousIndexEntry = segmentIndex.lastIndexEntry();
42 | if (optionalPreviousIndexEntry.isPresent()) {
43 | SegmentIndexEntry previousSegmentIndexEntry = optionalPreviousIndexEntry.get();
44 | long position = previousSegmentIndexEntry.recordFilePosition() + previousSegmentIndexEntry.recordByteLength();
45 | recordOutputStream.getChannel().position(position);
46 | } else {
47 | recordOutputStream.getChannel().position(1);
48 | }
49 | }
50 | }
51 |
52 | public long lastWrittenOffset() {
53 | return segmentIndex.lastIndexEntry().map(SegmentIndexEntry::getOffset).orElse(-1L);
54 | }
55 |
56 | public void append(Record record) throws IOException, SegmentIndex.IndexException, SegmentException {
57 | if (!record.topic().equals(topic)) {
58 | throw new SegmentException("Trying to append to wrong topic!\n" +
59 | "Expected topic: " + topic + " given topic: " + record.topic());
60 | }
61 | if (record.kafkaPartition() != partition) {
62 | throw new SegmentException("Trying to append to wrong partition!\n" +
63 | "Expected partition: " + partition + " given partition: " + partition);
64 | }
65 | if (record.kafkaOffset() < startOffset) {
66 | throw new SegmentException("Try to append a record with an offset smaller than the start offset. Something is very wrong. \n" +
67 | "Topic: " + record.topic() + "Partition: " + record.kafkaPartition() + " StartOffset: " + startOffset + " RecordOffset: " + record.kafkaOffset() + "\n" +
68 | "You probably forgot to delete a previous Backup\n");
69 | }
70 | if (record.kafkaOffset() <= lastWrittenOffset()) {
71 | // We are handling the offsets ourselves. This should never happen!
72 | throw new SegmentException("Trying to override a written record. There is something terribly wrong in your setup! Please check whether you are trying to override an existing backup" +
73 | "Topic: " + record.topic() + "Partition: " + record.kafkaPartition() + " lastWrittenOffset: " + lastWrittenOffset() + " RecordOffset: " + record.kafkaOffset());
74 | }
75 | long startPosition = recordOutputStream.getChannel().position();
76 | RecordSerde.write(recordOutputStream, record);
77 | long recordByteLength = recordOutputStream.getChannel().position() - startPosition;
78 | SegmentIndexEntry segmentIndexEntry = new SegmentIndexEntry(record.kafkaOffset(), startPosition, recordByteLength);
79 | segmentIndex.addEntry(segmentIndexEntry);
80 | }
81 |
82 | public String filePrefix() {
83 | return SegmentUtils.filePrefix(partition, startOffset);
84 | }
85 |
86 | public long size() throws IOException {
87 | return recordOutputStream.getChannel().size();
88 | }
89 |
90 | public void flush() throws IOException {
91 | recordOutputStream.flush();
92 | segmentIndex.flush();
93 | }
94 |
95 | public void close() throws IOException {
96 | recordOutputStream.close();
97 | segmentIndex.close();
98 | }
99 |
100 | public static class SegmentException extends Exception {
101 | SegmentException(String message) {
102 | super(message);
103 | }
104 | }
105 | }
106 |
--------------------------------------------------------------------------------
/src/main/java/de/azapps/kafkabackup/common/segment/UnverifiedSegmentReader.java:
--------------------------------------------------------------------------------
1 | package de.azapps.kafkabackup.common.segment;
2 |
3 | import de.azapps.kafkabackup.common.record.Record;
4 | import de.azapps.kafkabackup.common.record.RecordSerde;
5 |
6 | import java.io.EOFException;
7 | import java.io.FileInputStream;
8 | import java.io.IOException;
9 | import java.nio.file.Path;
10 | import java.util.ArrayList;
11 | import java.util.List;
12 |
13 | public class UnverifiedSegmentReader {
14 | private String topic;
15 | private int partition;
16 | private FileInputStream recordInputStream;
17 |
18 | public UnverifiedSegmentReader(Path recordFile) throws IOException {
19 | this(recordFile, "topic", 0);
20 | }
21 |
22 | public UnverifiedSegmentReader(Path recordFile, String topic, int partition) throws IOException {
23 | recordInputStream = new FileInputStream(recordFile.toFile());
24 | this.topic = topic;
25 | this.partition = partition;
26 | SegmentUtils.ensureValidSegment(recordInputStream);
27 | }
28 |
29 | public Record read() throws IOException {
30 | return RecordSerde.read(topic, partition, recordInputStream);
31 | }
32 |
33 | public List readN(int n) throws IOException {
34 | List records = new ArrayList<>(n);
35 | while (records.size() <= n) {
36 | try {
37 | Record record = read();
38 | records.add(record);
39 | } catch (EOFException e) {
40 | break;
41 | }
42 | }
43 | return records;
44 | }
45 |
46 | public List readFully() throws IOException {
47 | List records = new ArrayList<>();
48 | while (true) {
49 | try {
50 | Record record = read();
51 | records.add(record);
52 | } catch (EOFException e) {
53 | break;
54 | }
55 | }
56 | return records;
57 | }
58 |
59 | public long position() throws IOException {
60 | return recordInputStream.getChannel().position();
61 | }
62 |
63 | public void close() throws IOException {
64 | recordInputStream.close();
65 | }
66 | }
67 |
--------------------------------------------------------------------------------
/src/main/java/de/azapps/kafkabackup/sink/BackupSinkConfig.java:
--------------------------------------------------------------------------------
1 | package de.azapps.kafkabackup.sink;
2 |
3 | import org.apache.kafka.common.config.AbstractConfig;
4 | import org.apache.kafka.common.config.ConfigDef;
5 |
6 | import java.util.HashMap;
7 | import java.util.Map;
8 |
9 | class BackupSinkConfig extends AbstractConfig {
10 | static final String CLUSTER_PREFIX = "cluster.";
11 | static final String CLUSTER_BOOTSTRAP_SERVERS = CLUSTER_PREFIX + "bootstrap.servers";
12 | static final String ADMIN_CLIENT_PREFIX = "admin.";
13 | static final String TARGET_DIR_CONFIG = "target.dir";
14 | static final String MAX_SEGMENT_SIZE = "max.segment.size.bytes";
15 | static final String SNAPSHOT = "snapshot";
16 |
17 | static final ConfigDef CONFIG_DEF = new ConfigDef()
18 | .define(TARGET_DIR_CONFIG, ConfigDef.Type.STRING,
19 | ConfigDef.Importance.HIGH, "TargetDir")
20 | .define(MAX_SEGMENT_SIZE, ConfigDef.Type.INT, 1024 ^ 3, // 1 GiB
21 | ConfigDef.Importance.LOW, "Maximum segment size")
22 | .define(SNAPSHOT, ConfigDef.Type.BOOLEAN, false,
23 | ConfigDef.Importance.LOW, "Creates a snapshot. Terminates connector when end of all partitions has been reached.");
24 |
25 | BackupSinkConfig(Map props) {
26 | super(CONFIG_DEF, props, true);
27 | if (!props.containsKey(TARGET_DIR_CONFIG)) {
28 | throw new RuntimeException("Missing Configuration Variable: " + TARGET_DIR_CONFIG);
29 | }
30 | if (!props.containsKey(MAX_SEGMENT_SIZE)) {
31 | throw new RuntimeException("Missing Configuration Variable: " + MAX_SEGMENT_SIZE);
32 | }
33 | }
34 |
35 | Map adminConfig() {
36 | Map props = new HashMap<>();
37 | props.putAll(originalsWithPrefix(CLUSTER_PREFIX));
38 | props.putAll(originalsWithPrefix(ADMIN_CLIENT_PREFIX));
39 | return props;
40 | }
41 |
42 | String targetDir() {
43 | return getString(TARGET_DIR_CONFIG);
44 | }
45 |
46 | Integer maxSegmentSizeBytes() {
47 | return getInt(MAX_SEGMENT_SIZE);
48 | }
49 |
50 | Boolean snapShotMode() { return getBoolean(SNAPSHOT); }
51 |
52 | Map consumerConfig() {
53 | return new HashMap<>(originalsWithPrefix(CLUSTER_PREFIX));
54 | }
55 |
56 | }
57 |
--------------------------------------------------------------------------------
/src/main/java/de/azapps/kafkabackup/sink/BackupSinkConnector.java:
--------------------------------------------------------------------------------
1 | package de.azapps.kafkabackup.sink;
2 |
3 | import org.apache.kafka.common.config.ConfigDef;
4 | import org.apache.kafka.connect.connector.Task;
5 | import org.apache.kafka.connect.errors.ConnectException;
6 | import org.apache.kafka.connect.sink.SinkConnector;
7 |
8 | import java.util.ArrayList;
9 | import java.util.List;
10 | import java.util.Map;
11 |
12 | public class BackupSinkConnector extends SinkConnector {
13 | private Map config;
14 |
15 | @Override
16 | public void start(Map props) {
17 | config = props;
18 | }
19 |
20 | @Override
21 | public Class taskClass() {
22 | return BackupSinkTask.class;
23 | }
24 |
25 | @Override
26 | public List> taskConfigs(int maxTasks) {
27 | if (maxTasks > 1) {
28 | throw new ConnectException("kafka-backup can currently handle only one task.");
29 | }
30 | List> configs = new ArrayList<>();
31 | configs.add(config);
32 | return configs;
33 | }
34 |
35 | @Override
36 | public void stop() {
37 |
38 | }
39 |
40 | @Override
41 | public ConfigDef config() {
42 | return BackupSinkConfig.CONFIG_DEF;
43 | }
44 |
45 | @Override
46 | public String version() {
47 | return "0.1";
48 | }
49 | }
50 |
--------------------------------------------------------------------------------
/src/main/java/de/azapps/kafkabackup/source/BackupSourceConfig.java:
--------------------------------------------------------------------------------
1 | package de.azapps.kafkabackup.source;
2 |
3 | import de.azapps.kafkabackup.common.BackupConfig;
4 | import org.apache.kafka.common.config.AbstractConfig;
5 | import org.apache.kafka.common.config.ConfigDef;
6 |
7 | import java.util.Arrays;
8 | import java.util.HashMap;
9 | import java.util.List;
10 | import java.util.Map;
11 |
12 | class BackupSourceConfig extends BackupConfig {
13 | private static final String CLUSTER_KEY_DESERIALIZER = CLUSTER_PREFIX + "key.deserializer";
14 | private static final String CLUSTER_VALUE_DESERIALIZER = CLUSTER_PREFIX + "value.deserializer";
15 | private static final String BATCH_SIZE_CONFIG = "batch.size";
16 | private static final String SOURCE_DIR_CONFIG = "source.dir";
17 | private static final String TOPICS_CONFIG = "topics";
18 | static final String ALLOW_OLD_KAFKA_CONNECT_VERSION = "allow.old.kafka.connect.version.unsafe";
19 |
20 |
21 | private static final ConfigDef CONFIG_DEF = new ConfigDef()
22 | .define(SOURCE_DIR_CONFIG, ConfigDef.Type.STRING,
23 | ConfigDef.Importance.HIGH, "TargetDir")
24 | .define(BATCH_SIZE_CONFIG, ConfigDef.Type.INT, 100,
25 | ConfigDef.Importance.LOW, "Batch size per partition")
26 | .define(TOPICS_CONFIG, ConfigDef.Type.STRING,
27 | ConfigDef.Importance.HIGH, "Topics to restore");
28 |
29 | BackupSourceConfig(Map props) {
30 | super(CONFIG_DEF, props);
31 | if (!props.containsKey(SOURCE_DIR_CONFIG)) {
32 | throw new RuntimeException("Missing Configuration Variable: " + SOURCE_DIR_CONFIG);
33 | }
34 | if (!props.containsKey(TOPICS_CONFIG)) {
35 | throw new RuntimeException("Missing Configuration Variable: " + TOPICS_CONFIG);
36 | }
37 | if (!props.containsKey(CLUSTER_KEY_DESERIALIZER)) {
38 | throw new RuntimeException("Missing Configuration Variable: " + CLUSTER_KEY_DESERIALIZER);
39 | }
40 | if (!props.containsKey(CLUSTER_VALUE_DESERIALIZER)) {
41 | throw new RuntimeException("Missing Configuration Variable: " + CLUSTER_VALUE_DESERIALIZER);
42 | }
43 | }
44 |
45 | Map consumerConfig() {
46 | return new HashMap<>(originalsWithPrefix(CLUSTER_PREFIX));
47 | }
48 |
49 | String sourceDir() {
50 | return getString(SOURCE_DIR_CONFIG);
51 | }
52 |
53 | Integer batchSize() {
54 | return getInt(BATCH_SIZE_CONFIG);
55 | }
56 |
57 | List topics() {
58 | return Arrays.asList(getString(TOPICS_CONFIG).split("\\s*,\\s*"));
59 | }
60 |
61 |
62 | }
63 |
64 |
--------------------------------------------------------------------------------
/src/main/java/de/azapps/kafkabackup/source/BackupSourceConnector.java:
--------------------------------------------------------------------------------
1 | package de.azapps.kafkabackup.source;
2 |
3 | import org.apache.kafka.clients.producer.RecordMetadata;
4 | import org.apache.kafka.common.config.ConfigDef;
5 | import org.apache.kafka.connect.connector.Task;
6 | import org.apache.kafka.connect.errors.ConnectException;
7 | import org.apache.kafka.connect.source.SourceConnector;
8 | import org.apache.kafka.connect.source.SourceRecord;
9 | import org.apache.kafka.connect.source.SourceTask;
10 |
11 | import java.util.ArrayList;
12 | import java.util.List;
13 | import java.util.Map;
14 |
15 | public class BackupSourceConnector extends SourceConnector {
16 | private Map config;
17 |
18 |
19 | @Override
20 | public void start(Map props) {
21 | config = props;
22 | if (!config.getOrDefault(BackupSourceConfig.ALLOW_OLD_KAFKA_CONNECT_VERSION, "false").equals("true")) {
23 | try {
24 | SourceTask.class.getMethod("commitRecord", SourceRecord.class, RecordMetadata.class);
25 | } catch (NoSuchMethodException e) {
26 | throw new RuntimeException("Kafka Backup requires at least Kafka Connect 2.4. Otherwise Offsets cannot be committed. If you are sure what you are doing, please set " + BackupSourceConfig.ALLOW_OLD_KAFKA_CONNECT_VERSION + " to true");
27 | }
28 | }
29 | }
30 |
31 | @Override
32 | public Class taskClass() {
33 | return BackupSourceTask.class;
34 | }
35 |
36 | @Override
37 | public List> taskConfigs(int maxTasks) {
38 | if (maxTasks > 1) {
39 | throw new ConnectException("kafka-backup can currently handle only one task.");
40 | }
41 | List> configs = new ArrayList<>();
42 | configs.add(config);
43 | return configs;
44 | }
45 |
46 | @Override
47 | public void stop() {
48 |
49 | }
50 |
51 | @Override
52 | public ConfigDef config() {
53 | return new ConfigDef();
54 | }
55 |
56 | @Override
57 | public String version() {
58 | return "0.1";
59 | }
60 | }
61 |
--------------------------------------------------------------------------------
/src/test/assets/v1/partitionindex/testIndex:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/itadventurer/kafka-backup/4692ffeaf2f314aa9ad0d7a2346e47f24ab2dc3d/src/test/assets/v1/partitionindex/testIndex
--------------------------------------------------------------------------------
/src/test/assets/v1/records/empty_record:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/itadventurer/kafka-backup/4692ffeaf2f314aa9ad0d7a2346e47f24ab2dc3d/src/test/assets/v1/records/empty_record
--------------------------------------------------------------------------------
/src/test/assets/v1/records/header_record:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/itadventurer/kafka-backup/4692ffeaf2f314aa9ad0d7a2346e47f24ab2dc3d/src/test/assets/v1/records/header_record
--------------------------------------------------------------------------------
/src/test/assets/v1/records/null_record:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/itadventurer/kafka-backup/4692ffeaf2f314aa9ad0d7a2346e47f24ab2dc3d/src/test/assets/v1/records/null_record
--------------------------------------------------------------------------------
/src/test/assets/v1/records/simple_record:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/itadventurer/kafka-backup/4692ffeaf2f314aa9ad0d7a2346e47f24ab2dc3d/src/test/assets/v1/records/simple_record
--------------------------------------------------------------------------------
/src/test/assets/v1/segmentindex/testIndex:
--------------------------------------------------------------------------------
1 |
2 |
3 | $
4 |
--------------------------------------------------------------------------------
/src/test/assets/v1/segments/segment_partition_000_from_offset_0000000000_index:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/itadventurer/kafka-backup/4692ffeaf2f314aa9ad0d7a2346e47f24ab2dc3d/src/test/assets/v1/segments/segment_partition_000_from_offset_0000000000_index
--------------------------------------------------------------------------------
/src/test/assets/v1/segments/segment_partition_000_from_offset_0000000000_records:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/itadventurer/kafka-backup/4692ffeaf2f314aa9ad0d7a2346e47f24ab2dc3d/src/test/assets/v1/segments/segment_partition_000_from_offset_0000000000_records
--------------------------------------------------------------------------------
/src/test/java/de/azapps/kafkabackup/common/TestUtils.java:
--------------------------------------------------------------------------------
1 | package de.azapps.kafkabackup.common;
2 |
3 | import java.io.IOException;
4 | import java.nio.file.Files;
5 | import java.nio.file.Path;
6 | import java.nio.file.Paths;
7 |
8 | public class TestUtils {
9 | private static Path TEMP_DIR;
10 |
11 | static {
12 | try {
13 | TEMP_DIR = Files.createTempDirectory("kafka_backup_tests");
14 | } catch (IOException e) {
15 | e.printStackTrace();
16 | }
17 | }
18 |
19 | public static Path getTestDir(String tests) {
20 | Path ret = Paths.get(TEMP_DIR.toString(), tests);
21 | try {
22 | Files.createDirectories(ret);
23 | } catch (Exception e) {
24 | throw new RuntimeException(e);
25 | }
26 | return ret;
27 | }
28 | }
29 |
--------------------------------------------------------------------------------
/src/test/java/de/azapps/kafkabackup/common/partition/PartitionIndexTest.java:
--------------------------------------------------------------------------------
1 | package de.azapps.kafkabackup.common.partition;
2 |
3 | import de.azapps.kafkabackup.common.TestUtils;
4 | import de.azapps.kafkabackup.common.segment.SegmentIndex;
5 | import de.azapps.kafkabackup.common.segment.SegmentIndexEntry;
6 | import org.junit.jupiter.api.Test;
7 |
8 | import java.nio.file.Files;
9 | import java.nio.file.Path;
10 | import java.nio.file.Paths;
11 | import java.util.ArrayList;
12 | import java.util.List;
13 | import java.util.Optional;
14 |
15 | import static org.junit.jupiter.api.Assertions.*;
16 |
17 | public class PartitionIndexTest {
18 | private static Path TEMP_DIR = TestUtils.getTestDir("PartitionIndexTest");
19 |
20 | @Test
21 | public void simpleRoundtripTest() throws Exception {
22 | String indexFile = "simpleRoundtripTestIndex";
23 | List entries = new ArrayList<>();
24 | entries.add(new PartitionIndexEntry("s0", 0));
25 | entries.add(new PartitionIndexEntry("s100", 100));
26 | entries.add(new PartitionIndexEntry("s200", 200));
27 | entries.add(new PartitionIndexEntry("s300", 300));
28 | PartitionIndex index = new PartitionIndex(Paths.get(TEMP_DIR.toString(), indexFile));
29 | index.appendSegment(entries.get(0).filename(), entries.get(0).startOffset());
30 | index.appendSegment(entries.get(1).filename(), entries.get(1).startOffset());
31 | index.appendSegment(entries.get(2).filename(), entries.get(2).startOffset());
32 | index.appendSegment(entries.get(3).filename(), entries.get(3).startOffset());
33 |
34 | assertEquals(entries, index.index());
35 | index.close();
36 |
37 |
38 | PartitionIndex b = new PartitionIndex(Paths.get(TEMP_DIR.toString(), indexFile));
39 | assertEquals(entries, b.index());
40 | b.seek(10);
41 | assertEquals(entries.get(0).filename(), b.readFileName());
42 | assertTrue(b.hasMoreData());
43 | b.seek(200);
44 | assertEquals(entries.get(2).filename(), b.readFileName());
45 | assertTrue(b.hasMoreData());
46 | b.seek(310);
47 | assertEquals(entries.get(3).filename(), b.readFileName());
48 | assertFalse(b.hasMoreData());
49 | b.close();
50 | }
51 |
52 |
53 | @Test
54 | public void testReadV1Index() throws Exception {
55 | String indexFile = "testIndex";
56 | Path directory = Paths.get("src/test/assets/v1/partitionindex");
57 | List entries = new ArrayList<>();
58 | entries.add(new PartitionIndexEntry("s0", 0));
59 | entries.add(new PartitionIndexEntry("s100", 100));
60 | entries.add(new PartitionIndexEntry("s200", 200));
61 | entries.add(new PartitionIndexEntry("s300", 300));
62 |
63 | PartitionIndex b = new PartitionIndex(Paths.get(directory.toString(), indexFile));
64 | assertEquals(entries, b.index());
65 | }
66 |
67 |
68 | /**
69 | * Utility function to be run once when the format on disk changes to be able to stay backwards-compatible
70 | *
71 | * Call it manually once when the format changes
72 | */
73 | private static void writeTestIndexToFile() throws Exception {
74 | String indexFile = "testIndex";
75 | Path directory = Paths.get("src/test/assets/v1/partitionindex"); // CHANGEME WHEN CHANGING DATA FORMAT!
76 | Files.createDirectories(directory);
77 |
78 | List entries = new ArrayList<>();
79 | entries.add(new PartitionIndexEntry("s0", 0));
80 | entries.add(new PartitionIndexEntry("s100", 100));
81 | entries.add(new PartitionIndexEntry("s200", 200));
82 | entries.add(new PartitionIndexEntry("s300", 300));
83 |
84 | PartitionIndex index = new PartitionIndex(Paths.get(directory.toString(), indexFile));
85 | index.appendSegment(entries.get(0).filename(), entries.get(0).startOffset());
86 | index.appendSegment(entries.get(1).filename(), entries.get(1).startOffset());
87 | index.appendSegment(entries.get(2).filename(), entries.get(2).startOffset());
88 | index.appendSegment(entries.get(3).filename(), entries.get(3).startOffset());
89 | index.close();
90 | }
91 | }
92 |
--------------------------------------------------------------------------------
/src/test/java/de/azapps/kafkabackup/common/partition/PartitionSerdeTest.java:
--------------------------------------------------------------------------------
1 | package de.azapps.kafkabackup.common.partition;
2 |
3 | import de.azapps.kafkabackup.common.TestUtils;
4 | import de.azapps.kafkabackup.common.record.Record;
5 | import de.azapps.kafkabackup.common.segment.SegmentReader;
6 | import de.azapps.kafkabackup.common.segment.SegmentUtils;
7 | import org.apache.kafka.common.header.internals.RecordHeaders;
8 | import org.apache.kafka.common.record.TimestampType;
9 | import org.apache.kafka.connect.header.ConnectHeaders;
10 | import org.apache.kafka.connect.header.Headers;
11 | import org.junit.jupiter.api.Test;
12 |
13 | import java.nio.charset.StandardCharsets;
14 | import java.nio.file.Files;
15 | import java.nio.file.Path;
16 | import java.util.ArrayList;
17 | import java.util.List;
18 |
19 | import static org.junit.jupiter.api.Assertions.assertEquals;
20 | import static org.junit.jupiter.api.Assertions.assertFalse;
21 |
22 | public class PartitionSerdeTest {
23 | private static final String TOPIC = "test-topic";
24 | private static final byte[] KEY_BYTES = "test-key".getBytes(StandardCharsets.UTF_8);
25 | private static final byte[] VALUE_BYTES = "test-value".getBytes(StandardCharsets.UTF_8);
26 | private static final Path TEMP_DIR = TestUtils.getTestDir("PartitionSerdeTest");
27 |
28 | private static final RecordHeaders HEADERS = new RecordHeaders();
29 | private static final byte[] HEADER_0_VALUE_BYTES = "header0-value".getBytes(StandardCharsets.UTF_8);
30 | private static final byte[] HEADER_1_VALUE_BYTES = "header1-value".getBytes(StandardCharsets.UTF_8);
31 | static {
32 | HEADERS.add("", new byte[0]);
33 | HEADERS.add("null", null);
34 | HEADERS.add("value0", HEADER_0_VALUE_BYTES);
35 | HEADERS.add("value1", HEADER_1_VALUE_BYTES);
36 | }
37 | @Test
38 | public void simpleRoundtripTest() throws Exception {
39 | int partition = 0;
40 |
41 | List records = new ArrayList<>();
42 | records.add(new Record(TOPIC, partition, KEY_BYTES, VALUE_BYTES, 0));
43 | records.add(new Record(TOPIC, partition, null, null, 1));
44 | records.add(new Record(TOPIC, partition, new byte[0], new byte[0], 2));
45 | records.add(new Record(TOPIC, partition, KEY_BYTES, VALUE_BYTES, 3, null, TimestampType.NO_TIMESTAMP_TYPE, HEADERS));
46 |
47 | PartitionWriter partitionWriter = new PartitionWriter(TOPIC, partition, TEMP_DIR, 50);
48 | partitionWriter.append(records.get(0));
49 | partitionWriter.append(records.get(1));
50 | partitionWriter.append(records.get(2));
51 | partitionWriter.append(records.get(3));
52 | partitionWriter.close();
53 |
54 | PartitionReader partitionReader = new PartitionReader(TOPIC, partition, TEMP_DIR);
55 | assertEquals(records, partitionReader.readFully());
56 | assertFalse(partitionReader.hasMoreData());
57 | partitionReader.seek(1);
58 | assertEquals(records.get(1), partitionReader.read());
59 | partitionReader.seek(3);
60 | assertEquals(records.get(3), partitionReader.read());
61 | assertFalse(partitionReader.hasMoreData());
62 | }
63 |
64 | @Test
65 | public void smallSegmentSizeTest() throws Exception {
66 | int partition = 1;
67 |
68 | List records = new ArrayList<>();
69 | records.add(new Record(TOPIC, partition, KEY_BYTES, VALUE_BYTES, 0));
70 | records.add(new Record(TOPIC, partition, null, null, 1));
71 | records.add(new Record(TOPIC, partition, new byte[0], new byte[0], 2));
72 |
73 | PartitionWriter partitionWriter = new PartitionWriter(TOPIC, partition, TEMP_DIR, 1);
74 | partitionWriter.append(records.get(0));
75 | partitionWriter.append(records.get(1));
76 | partitionWriter.append(records.get(2));
77 | partitionWriter.close();
78 |
79 | SegmentReader a = new SegmentReader(TOPIC, partition, TEMP_DIR, 0);
80 | assertEquals(records.get(0), a.read());
81 | assertFalse(a.hasMoreData());
82 | SegmentReader b = new SegmentReader(TOPIC, partition, TEMP_DIR, 1);
83 | assertEquals(records.get(1), b.read());
84 | assertFalse(b.hasMoreData());
85 | SegmentReader c = new SegmentReader(TOPIC, partition, TEMP_DIR, 2);
86 | assertEquals(records.get(2), c.read());
87 | assertFalse(c.hasMoreData());
88 | }
89 |
90 | @Test
91 | public void deleteSomeSegmentsTest() throws Exception {
92 | int partition = 2;
93 |
94 | List records = new ArrayList<>();
95 | records.add(new Record(TOPIC, partition, KEY_BYTES, VALUE_BYTES, 0));
96 | records.add(new Record(TOPIC, partition, null, null, 1));
97 | records.add(new Record(TOPIC, partition, new byte[0], new byte[0], 2));
98 | records.add(new Record(TOPIC, partition, KEY_BYTES, VALUE_BYTES, 3, null, TimestampType.NO_TIMESTAMP_TYPE, HEADERS));
99 |
100 | PartitionWriter partitionWriter = new PartitionWriter(TOPIC, partition, TEMP_DIR, 1);
101 | partitionWriter.append(records.get(0));
102 | partitionWriter.append(records.get(1));
103 | partitionWriter.append(records.get(2));
104 | partitionWriter.append(records.get(3));
105 | partitionWriter.close();
106 |
107 | // Delete segments 0 and 2
108 | Files.delete(SegmentUtils.recordsFile(TEMP_DIR, partition, 0));
109 | Files.delete(SegmentUtils.indexFile(TEMP_DIR, partition, 0));
110 | Files.delete(SegmentUtils.recordsFile(TEMP_DIR, partition, 2));
111 | Files.delete(SegmentUtils.indexFile(TEMP_DIR, partition, 2));
112 | Files.delete(PartitionUtils.indexFile(TEMP_DIR, partition));
113 |
114 | // Restore indices
115 | PartitionIndexRestore restore = new PartitionIndexRestore(TEMP_DIR, partition);
116 | restore.restore();
117 |
118 | // Expected
119 | List expected = new ArrayList<>();
120 | expected.add(records.get(1));
121 | expected.add(records.get(3));
122 |
123 | PartitionReader reader = new PartitionReader(TOPIC, partition, TEMP_DIR);
124 | assertEquals(expected, reader.readFully());
125 | }
126 | }
127 |
--------------------------------------------------------------------------------
/src/test/java/de/azapps/kafkabackup/common/record/RecordSerdeTest.java:
--------------------------------------------------------------------------------
1 | package de.azapps.kafkabackup.common.record;
2 |
3 | import org.apache.kafka.common.header.internals.RecordHeaders;
4 | import org.apache.kafka.common.record.TimestampType;
5 | import org.junit.jupiter.api.Test;
6 |
7 | import java.io.*;
8 | import java.nio.charset.StandardCharsets;
9 |
10 | import static org.junit.jupiter.api.Assertions.assertEquals;
11 | import static org.junit.jupiter.api.Assertions.assertNotEquals;
12 |
13 | public class RecordSerdeTest {
14 |
15 | private static final String TOPIC = "test-topic";
16 | private static final int PARTITION = 42;
17 | private static final long OFFSET = 123;
18 | private static final byte[] KEY_BYTES = "test-key".getBytes(StandardCharsets.UTF_8);
19 | private static final byte[] VALUE_BYTES = "test-value".getBytes(StandardCharsets.UTF_8);
20 | private static final byte[] NULL_TIMESTAMP_BYTES = "null-timestamp".getBytes(StandardCharsets.UTF_8);
21 |
22 | private static final String SIMPLE_RECORD_FILE = "simple_record";
23 | private static final String NULL_RECORD_FILE = "null_record";
24 | private static final String EMPTY_RECORD_FILE = "empty_record";
25 | private static final String HEADER_RECORD_FILE = "header_record";
26 |
27 | // Example records
28 | private static final Record SIMPLE_RECORD, NULL_RECORD, EMPTY_RECORD, HEADER_RECORD, NULL_TIMESTAMP_RECORD;
29 |
30 | static {
31 | SIMPLE_RECORD = new Record(TOPIC, PARTITION, KEY_BYTES, VALUE_BYTES, OFFSET);
32 | NULL_RECORD = new Record(TOPIC, PARTITION, null, null, OFFSET);
33 | EMPTY_RECORD = new Record(TOPIC, PARTITION, new byte[0], new byte[0], OFFSET);
34 | NULL_TIMESTAMP_RECORD = new Record(TOPIC, PARTITION, NULL_TIMESTAMP_BYTES, null, OFFSET, null, TimestampType.CREATE_TIME);
35 | // Build multiple headers that might cause problems
36 | RecordHeaders headers = new RecordHeaders();
37 | headers.add("", new byte[0]);
38 | headers.add("null", null);
39 | headers.add("value", VALUE_BYTES);
40 | HEADER_RECORD = new Record(TOPIC, PARTITION, KEY_BYTES, VALUE_BYTES, OFFSET, null, TimestampType.NO_TIMESTAMP_TYPE, headers);
41 | }
42 |
43 | @Test
44 | public void roundtripTest() throws Exception {
45 | Record simpleRoundtrip = writeAndReadRecord(SIMPLE_RECORD);
46 | assertEquals(SIMPLE_RECORD, simpleRoundtrip);
47 | }
48 |
49 | @Test
50 | public void roundtripWithNull() throws Exception {
51 | Record nullRoundtrip = writeAndReadRecord(NULL_RECORD);
52 | assertEquals(NULL_RECORD, nullRoundtrip);
53 |
54 | Record emptyRoundtrip = writeAndReadRecord(EMPTY_RECORD);
55 | assertEquals(EMPTY_RECORD, emptyRoundtrip);
56 |
57 | // Must be different
58 | assertNotEquals(nullRoundtrip, emptyRoundtrip);
59 | }
60 |
61 | @Test
62 | public void roundtripNullTimestamp() throws Exception {
63 | Record nullTimestampRoundtrip =writeAndReadRecord(NULL_TIMESTAMP_RECORD);
64 | assertEquals(NULL_TIMESTAMP_RECORD, nullTimestampRoundtrip);
65 | }
66 |
67 | @Test
68 | public void roundtripHeaders() throws Exception {
69 | Record headerRoundtrip = writeAndReadRecord(HEADER_RECORD);
70 | assertEquals(HEADER_RECORD, headerRoundtrip);
71 | }
72 |
73 | /**
74 | * DO NOT CHANGE THIS TEST!
75 | */
76 | @Test
77 | public void readV1() throws Exception {
78 | File v1Directory = new File("src/test/assets/v1/records");
79 | Record simpleRecord = readFromFile(new File(v1Directory, SIMPLE_RECORD_FILE));
80 | assertEquals(SIMPLE_RECORD, simpleRecord);
81 | Record nullRecord = readFromFile(new File(v1Directory, NULL_RECORD_FILE));
82 | assertEquals(NULL_RECORD, nullRecord);
83 | assertNotEquals(SIMPLE_RECORD, nullRecord); // just to make sure!
84 | Record emptyRecord = readFromFile(new File(v1Directory, EMPTY_RECORD_FILE));
85 | assertEquals(EMPTY_RECORD, emptyRecord);
86 | assertNotEquals(NULL_RECORD, emptyRecord); // just to make sure!
87 | Record headerRecord = readFromFile(new File(v1Directory, HEADER_RECORD_FILE));
88 | assertEquals(HEADER_RECORD, headerRecord);
89 | assertNotEquals(EMPTY_RECORD, headerRecord); // just to make sure!
90 | }
91 |
92 | // UTILS
93 |
94 | private Record writeAndReadRecord(Record record) throws IOException {
95 | ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
96 | RecordSerde.write(outputStream, record);
97 | byte[] data = outputStream.toByteArray();
98 | return RecordSerde.read(TOPIC, PARTITION, new ByteArrayInputStream(data));
99 | }
100 |
101 | private static Record readFromFile(File file) throws IOException {
102 | FileInputStream inputStream = new FileInputStream(file);
103 | return RecordSerde.read(TOPIC, PARTITION, inputStream);
104 | }
105 |
106 | /**
107 | * Utility function to be run once when the format on disk changes to be able to stay backwards-compatible
108 | *
109 | * Call it manually once when the format changes
110 | */
111 | private static void writeTestRecordsToFile() throws IOException {
112 | File directory = new File("src/test/assets/v1/records"); // CHANGEME WHEN CHANGING DATA FORMAT!
113 | writeCurrentVersionRecordToFile(SIMPLE_RECORD, new File(directory, SIMPLE_RECORD_FILE));
114 | writeCurrentVersionRecordToFile(NULL_RECORD, new File(directory, NULL_RECORD_FILE));
115 | writeCurrentVersionRecordToFile(EMPTY_RECORD, new File(directory, EMPTY_RECORD_FILE));
116 | writeCurrentVersionRecordToFile(HEADER_RECORD, new File(directory, HEADER_RECORD_FILE));
117 | }
118 |
119 | private static void writeCurrentVersionRecordToFile(Record record, File file) throws IOException {
120 | FileOutputStream outputStream = new FileOutputStream(file);
121 | RecordSerde.write(outputStream, record);
122 | }
123 | }
124 |
--------------------------------------------------------------------------------
/src/test/java/de/azapps/kafkabackup/common/record/RecordTest.java:
--------------------------------------------------------------------------------
1 | package de.azapps.kafkabackup.common.record;
2 |
3 | import org.apache.kafka.common.header.internals.RecordHeaders;
4 | import org.apache.kafka.common.record.TimestampType;
5 | import org.apache.kafka.connect.sink.SinkRecord;
6 | import org.junit.jupiter.api.Test;
7 |
8 | import java.nio.charset.StandardCharsets;
9 | import java.util.Arrays;
10 |
11 | import static org.junit.jupiter.api.Assertions.assertEquals;
12 | import static org.junit.jupiter.api.Assertions.assertNotEquals;
13 |
14 | public class RecordTest {
15 | private static final String TOPIC = "test-topic";
16 | private static final int PARTITION = 42;
17 | private static final long OFFSET = 123;
18 | private static final TimestampType TIMESTAMP_TYPE = TimestampType.LOG_APPEND_TIME;
19 | private static final Long TIMESTAMP = 573831430000L;
20 | // encoding here is not really important, we just want some bytes
21 | private static final byte[] KEY_BYTES = "test-key".getBytes(StandardCharsets.UTF_8);
22 | private static final byte[] VALUE_BYTES = "test-value".getBytes(StandardCharsets.UTF_8);
23 | // Header fixtures:
24 | private static final byte[] HEADER_0_VALUE_BYTES = "header0-value".getBytes(StandardCharsets.UTF_8);
25 | private static final byte[] HEADER_1_VALUE_BYTES = "header1-value".getBytes(StandardCharsets.UTF_8);
26 | private static final RecordHeaders HEADERS = new RecordHeaders();
27 |
28 | static {
29 | HEADERS.add("", new byte[0]);
30 | HEADERS.add("null", null);
31 | HEADERS.add("value0", HEADER_0_VALUE_BYTES);
32 | HEADERS.add("value1", HEADER_1_VALUE_BYTES);
33 | }
34 |
35 | @Test
36 | public void equalsIdentityTrueTest() {
37 | // GIVEN
38 | Record a = new Record(TOPIC, PARTITION, KEY_BYTES, VALUE_BYTES, OFFSET, TIMESTAMP, TIMESTAMP_TYPE, HEADERS);
39 |
40 | // THEN
41 | assertEquals(a, a);
42 | }
43 |
44 | @Test
45 | public void equalsValueTrueTest() {
46 | // GIVEN
47 | Record a = new Record(TOPIC, PARTITION, KEY_BYTES, VALUE_BYTES, OFFSET, TIMESTAMP, TIMESTAMP_TYPE, HEADERS);
48 |
49 | RecordHeaders bHeaders = new RecordHeaders();
50 | bHeaders.add("", new byte[0]);
51 | bHeaders.add("null", null);
52 | bHeaders.add("value0", Arrays.copyOf(HEADER_0_VALUE_BYTES, HEADER_0_VALUE_BYTES.length));
53 | bHeaders.add("value1", Arrays.copyOf(HEADER_1_VALUE_BYTES, HEADER_1_VALUE_BYTES.length));
54 | Record b = new Record(TOPIC, PARTITION, KEY_BYTES, VALUE_BYTES, OFFSET, TIMESTAMP, TIMESTAMP_TYPE, bHeaders);
55 |
56 | // THEN
57 | assertEquals(a, b);
58 | assertEquals(b, a);
59 | }
60 |
61 | @Test
62 | public void equalsFalseBecauseStrictSubsetTest() {
63 | // GIVEN
64 | Record a = new Record(TOPIC, PARTITION, KEY_BYTES, VALUE_BYTES, OFFSET, TIMESTAMP, TIMESTAMP_TYPE, HEADERS);
65 | Record b = new Record(TOPIC, PARTITION, KEY_BYTES, VALUE_BYTES, OFFSET);
66 |
67 | // THEN
68 | assertNotEquals(a, b);
69 | assertNotEquals(b, a);
70 | }
71 |
72 | @Test
73 | public void equalsFalseBecauseHeadersStrictSubsetTest() {
74 | // GIVEN
75 | RecordHeaders aHeaders = new RecordHeaders();
76 | aHeaders.add("header0-key", Arrays.copyOf(HEADER_0_VALUE_BYTES, HEADER_0_VALUE_BYTES.length));
77 | aHeaders.add("header1-key", Arrays.copyOf(HEADER_1_VALUE_BYTES, HEADER_1_VALUE_BYTES.length));
78 | Record a = new Record(TOPIC, PARTITION, KEY_BYTES, VALUE_BYTES, OFFSET, TIMESTAMP, TIMESTAMP_TYPE, aHeaders);
79 |
80 | RecordHeaders bHeaders = new RecordHeaders();
81 | bHeaders.add("header0-key", Arrays.copyOf(HEADER_0_VALUE_BYTES, HEADER_0_VALUE_BYTES.length));
82 | Record b = new Record(TOPIC, PARTITION, KEY_BYTES, VALUE_BYTES, OFFSET, TIMESTAMP, TIMESTAMP_TYPE, bHeaders);
83 |
84 |
85 | RecordHeaders cHeaders = new RecordHeaders();
86 | cHeaders.add("header1-key", Arrays.copyOf(HEADER_0_VALUE_BYTES, HEADER_0_VALUE_BYTES.length));
87 | cHeaders.add("header1-key", Arrays.copyOf(HEADER_1_VALUE_BYTES, HEADER_1_VALUE_BYTES.length));
88 | Record c = new Record(TOPIC, PARTITION, KEY_BYTES, VALUE_BYTES, OFFSET, TIMESTAMP, TIMESTAMP_TYPE, cHeaders);
89 |
90 | // THEN
91 | assertNotEquals(a, b);
92 | assertNotEquals(b, a);
93 | assertNotEquals(a, c);
94 | assertNotEquals(b, c);
95 | }
96 |
97 | /**
98 | * This is not used during normal operations, but we need to verify that this works
99 | * correctly as we use the functions for our end to end tests!
100 | */
101 | @Test
102 | public void roundtripSinkRecordTest() {
103 |
104 | // given
105 | Record a = new Record(TOPIC, PARTITION, KEY_BYTES, VALUE_BYTES, OFFSET, TIMESTAMP, TIMESTAMP_TYPE, HEADERS);
106 | Record b = new Record(TOPIC, PARTITION, KEY_BYTES, VALUE_BYTES, 3, null, TimestampType.NO_TIMESTAMP_TYPE, HEADERS);
107 | Record c = new Record(TOPIC, PARTITION, KEY_BYTES, VALUE_BYTES, 0);
108 | Record d = new Record(TOPIC, PARTITION, null, null, 1);
109 | Record e = new Record(TOPIC, PARTITION, new byte[0], new byte[0], 2);
110 | Record f = new Record(TOPIC, PARTITION, KEY_BYTES, VALUE_BYTES, OFFSET);
111 |
112 | // transform
113 | SinkRecord srA = a.toSinkRecord();
114 | SinkRecord srB = b.toSinkRecord();
115 | SinkRecord srC = c.toSinkRecord();
116 | SinkRecord srD = d.toSinkRecord();
117 | SinkRecord srE = e.toSinkRecord();
118 | SinkRecord srF = f.toSinkRecord();
119 |
120 | // expect
121 | assertEquals(a, Record.fromSinkRecord(srA));
122 | assertEquals(b, Record.fromSinkRecord(srB));
123 | assertEquals(c, Record.fromSinkRecord(srC));
124 | assertEquals(d, Record.fromSinkRecord(srD));
125 | assertEquals(e, Record.fromSinkRecord(srE));
126 | assertEquals(f, Record.fromSinkRecord(srF));
127 |
128 |
129 | }
130 | }
131 |
--------------------------------------------------------------------------------
/src/test/java/de/azapps/kafkabackup/common/segment/SegmentIndexTest.java:
--------------------------------------------------------------------------------
1 | package de.azapps.kafkabackup.common.segment;
2 |
3 | import de.azapps.kafkabackup.common.TestUtils;
4 | import de.azapps.kafkabackup.common.record.Record;
5 | import org.apache.kafka.common.record.TimestampType;
6 | import org.junit.jupiter.api.Test;
7 |
8 | import java.nio.charset.StandardCharsets;
9 | import java.nio.file.Files;
10 | import java.nio.file.Path;
11 | import java.nio.file.Paths;
12 | import java.util.ArrayList;
13 | import java.util.List;
14 | import java.util.Optional;
15 |
16 | import static org.junit.jupiter.api.Assertions.*;
17 |
18 | public class SegmentIndexTest {
19 | private static final String TOPIC = "test-topic";
20 | private static final byte[] KEY_BYTES = "test-key".getBytes(StandardCharsets.UTF_8);
21 | private static final byte[] VALUE_BYTES = "test-value".getBytes(StandardCharsets.UTF_8);
22 | private static final SegmentIndexEntry ENTRY1 = new SegmentIndexEntry(0, 1, 10);
23 | private static final SegmentIndexEntry ENTRY2 = new SegmentIndexEntry(1, 11, 10);
24 | private static final SegmentIndexEntry ENTRY3 = new SegmentIndexEntry(5, 21, 15);
25 | private static final SegmentIndexEntry ENTRY4 = new SegmentIndexEntry(6, 36, 10);
26 | private static final Path TEMP_DIR = TestUtils.getTestDir("SegmentIndexTest");
27 |
28 | @Test
29 | public void simpleRoundtripTest() throws Exception {
30 | String indexFile = "simpleRoundtripTestIndex";
31 | SegmentIndex index = new SegmentIndex(Paths.get(TEMP_DIR.toString(), indexFile));
32 | assertEquals(0L, index.lastValidStartPosition());
33 | List entries = new ArrayList<>();
34 | entries.add(ENTRY1);
35 | index.addEntry(ENTRY1);
36 | entries.add(ENTRY2);
37 | index.addEntry(ENTRY2);
38 | entries.add(ENTRY3);
39 | index.addEntry(ENTRY3);
40 | entries.add(ENTRY4);
41 | index.addEntry(ENTRY4);
42 |
43 | index.close();
44 |
45 |
46 | SegmentIndex b = new SegmentIndex(Paths.get(TEMP_DIR.toString(), indexFile));
47 | assertEquals(entries, b.index());
48 | assertEquals(Optional.of(ENTRY3.recordFilePosition()), b.findByOffset(5));
49 | assertEquals(Optional.of(ENTRY3.recordFilePosition()), b.findEarliestWithHigherOrEqualOffset(2));
50 | assertEquals(Optional.empty(), b.findEarliestWithHigherOrEqualOffset(11));
51 | assertEquals(36, b.lastValidStartPosition());
52 | }
53 |
54 | @Test
55 | public void writeRecordThenCheckIndex() throws Exception {
56 | int partition = 0;
57 | SegmentWriter segmentWriter = new SegmentWriter(TOPIC, partition, 0, TEMP_DIR);
58 | segmentWriter.append(new Record(TOPIC, partition, KEY_BYTES, VALUE_BYTES, 0));
59 |
60 | SegmentIndex i1 = new SegmentIndex(SegmentUtils.indexFile(TEMP_DIR, partition, 0));
61 | assertEquals(1, i1.size());
62 |
63 |
64 | segmentWriter.append(new Record(TOPIC, partition, null, null, 1));
65 | segmentWriter.append(new Record(TOPIC, partition, new byte[0], new byte[0], 2));
66 | segmentWriter.close();
67 |
68 | SegmentIndex i2 = new SegmentIndex(SegmentUtils.indexFile(TEMP_DIR, partition, 0));
69 | assertEquals(3, i2.size());
70 | long fileLength = SegmentUtils.recordsFile(TEMP_DIR, partition, 0).toFile().length();
71 | //noinspection OptionalGetWithoutIsPresent
72 | SegmentIndexEntry entry = i2.lastIndexEntry().get();
73 | assertEquals(fileLength, entry.recordFilePosition() + entry.recordByteLength());
74 | }
75 |
76 | @Test
77 | public void restoreTest() throws Exception {
78 | int partition = 1;
79 | List records = new ArrayList<>();
80 | records.add(new Record(TOPIC, partition, KEY_BYTES, VALUE_BYTES, 0));
81 | records.add(new Record(TOPIC, partition, null, null, 1));
82 | records.add(new Record(TOPIC, partition, new byte[0], new byte[0], 2));
83 | records.add(new Record(TOPIC, partition, KEY_BYTES, VALUE_BYTES, 3, null, TimestampType.NO_TIMESTAMP_TYPE));
84 |
85 | SegmentWriter segmentWriter = new SegmentWriter(TOPIC, partition, 0, TEMP_DIR);
86 | for (Record record : records) {
87 | segmentWriter.append(record);
88 | }
89 | segmentWriter.close();
90 | Path indexFile = SegmentUtils.indexFile(TEMP_DIR, partition, 0);
91 | SegmentIndex a = new SegmentIndex(indexFile);
92 | Files.delete(indexFile);
93 | SegmentIndexRestore restore = new SegmentIndexRestore(SegmentUtils.recordsFile(TEMP_DIR, partition, 0));
94 | restore.restore();
95 | SegmentIndex b = new SegmentIndex(indexFile);
96 | assertEquals(a.index(), b.index());
97 | }
98 |
99 | @Test
100 | public void incrementingIndex() throws Exception {
101 | String indexFile = "incrementingIndex";
102 | SegmentIndex index = new SegmentIndex(Paths.get(TEMP_DIR.toString(), indexFile));
103 | index.addEntry(new SegmentIndexEntry(5, 22, 15));
104 | // Wrong offset
105 | assertThrows(SegmentIndex.IndexException.class,
106 | () -> index.addEntry(new SegmentIndexEntry(0, 37, 10)));
107 | // Should be ok
108 | assertDoesNotThrow(() -> index.addEntry(new SegmentIndexEntry(10, 37, 10)));
109 | index.close();
110 | }
111 |
112 | @Test
113 | public void emptyIndexTest() throws Exception {
114 | String indexFile = "emptyIndexTest";
115 | SegmentIndex index = new SegmentIndex(Paths.get(TEMP_DIR.toString(), indexFile));
116 | assertEquals(0L, index.lastValidStartPosition());
117 | index.close();
118 |
119 |
120 | SegmentIndex b = new SegmentIndex(Paths.get(TEMP_DIR.toString(), indexFile));
121 | assertEquals(0L, b.lastValidStartPosition());
122 | assertEquals(Optional.empty(), b.findEarliestWithHigherOrEqualOffset(0));
123 | assertEquals(Optional.empty(), b.findEarliestWithHigherOrEqualOffset(11));
124 | assertTrue(b.index().isEmpty());
125 | }
126 |
127 |
128 | @Test
129 | public void testReadV1Index() throws Exception {
130 | String indexFile = "testIndex";
131 | Path directory = Paths.get("src/test/assets/v1/segmentindex");
132 | List entries = new ArrayList<>();
133 | entries.add(ENTRY1);
134 | entries.add(ENTRY2);
135 | entries.add(ENTRY3);
136 | entries.add(ENTRY4);
137 |
138 | SegmentIndex b = new SegmentIndex(Paths.get(directory.toString(), indexFile));
139 | assertEquals(entries, b.index());
140 | assertEquals(Optional.of(ENTRY3.recordFilePosition()), b.findByOffset(5));
141 | assertEquals(Optional.of(ENTRY3.recordFilePosition()), b.findEarliestWithHigherOrEqualOffset(2));
142 | assertEquals(36, b.lastValidStartPosition());
143 | }
144 |
145 |
146 | /**
147 | * Utility function to be run once when the format on disk changes to be able to stay backwards-compatible
148 | *
149 | * Call it manually once when the format changes
150 | */
151 | private static void writeTestIndexToFile() throws Exception {
152 | String indexFile = "testIndex";
153 | Path directory = Paths.get("src/test/assets/v1/segmentindex"); // CHANGEME WHEN CHANGING DATA FORMAT!
154 | Files.createDirectories(directory);
155 |
156 | SegmentIndex index = new SegmentIndex(Paths.get(directory.toString(), indexFile));
157 | index.addEntry(ENTRY1);
158 | index.addEntry(ENTRY2);
159 | index.addEntry(ENTRY3);
160 | index.addEntry(ENTRY4);
161 | index.close();
162 | }
163 | }
164 |
--------------------------------------------------------------------------------
/src/test/java/de/azapps/kafkabackup/sink/MockEndOffsetReader.java:
--------------------------------------------------------------------------------
1 | package de.azapps.kafkabackup.sink;
2 |
3 | import de.azapps.kafkabackup.common.offset.EndOffsetReader;
4 | import org.apache.kafka.common.TopicPartition;
5 |
6 | import java.util.Collection;
7 | import java.util.HashMap;
8 | import java.util.Map;
9 |
10 | public class MockEndOffsetReader extends EndOffsetReader {
11 | private Map offsets;
12 | public MockEndOffsetReader(Map offsets) {
13 | super(new HashMap<>());
14 | this.offsets = offsets;
15 | }
16 | @Override
17 | public Map getEndOffsets(Collection partitions) {
18 | return offsets;
19 | }
20 | }
21 |
--------------------------------------------------------------------------------
/src/test/java/de/azapps/kafkabackup/sink/MockOffsetSink.java:
--------------------------------------------------------------------------------
1 | package de.azapps.kafkabackup.sink;
2 |
3 | import de.azapps.kafkabackup.common.offset.OffsetSink;
4 | import org.apache.kafka.clients.admin.AdminClient;
5 |
6 | import java.io.IOException;
7 | import java.nio.file.Path;
8 |
9 | public class MockOffsetSink extends OffsetSink {
10 | public MockOffsetSink(AdminClient adminClient, Path targetDir) {
11 | super(adminClient, targetDir);
12 | }
13 |
14 | @Override
15 | public void syncConsumerGroups() {
16 |
17 | }
18 |
19 | @Override
20 | public void syncOffsets() throws IOException {
21 | }
22 |
23 | @Override
24 | public void flush() throws IOException {
25 | }
26 |
27 | @Override
28 | public void close() throws IOException {
29 | }
30 | }
31 |
32 |
--------------------------------------------------------------------------------
/src/test/java/de/azapps/kafkabackup/sink/MockSinkTaskContext.java:
--------------------------------------------------------------------------------
1 | package de.azapps.kafkabackup.sink;
2 |
3 | import org.apache.kafka.common.TopicPartition;
4 | import org.apache.kafka.connect.sink.SinkTaskContext;
5 |
6 | import java.util.HashMap;
7 | import java.util.Map;
8 | import java.util.Set;
9 |
10 | public class MockSinkTaskContext implements SinkTaskContext {
11 |
12 | @Override
13 | public Map configs() {
14 | return new HashMap<>();
15 | }
16 |
17 | @Override
18 | public void offset(Map offsets) {
19 |
20 | }
21 |
22 | @Override
23 | public void offset(TopicPartition tp, long offset) {
24 |
25 | }
26 |
27 | @Override
28 | public void timeout(long timeoutMs) {
29 |
30 | }
31 |
32 | @Override
33 | public Set assignment() {
34 | return null;
35 | }
36 |
37 | @Override
38 | public void pause(TopicPartition... partitions) {
39 |
40 | }
41 |
42 | @Override
43 | public void resume(TopicPartition... partitions) {
44 |
45 | }
46 |
47 | @Override
48 | public void requestCommit() {
49 |
50 | }
51 | }
52 |
--------------------------------------------------------------------------------
/system_test/.gitignore:
--------------------------------------------------------------------------------
1 | out
--------------------------------------------------------------------------------
/system_test/01_simple_roundtrip_test.yaml:
--------------------------------------------------------------------------------
1 | # Test description
2 | #
3 | # Create two topics:
4 | # * backup-test-1partition: 1 partition
5 | # * backup-test-empty-topic: 3 partitions
6 | #
7 | # * Fill the 1partition topic with 300 messages á 10KB each in every partition
8 | # * Consume the messages with three consumer groups: cg-100: consume 100 messages, cg-200 consume 200 messages, cg-300 consume 300 messages
9 | # * Fill one of the 3 partitions of empty-topic with 300 messages. Leave the other partitions empty
10 | #
11 | # * Take a backup
12 | # * Stop the Cluster
13 | #
14 | # * Start the backup
15 | # * Verify that all messages have been written correctly
16 | # * Check the consumer offsets
17 | #
18 | # * Stop everything
19 | - name: coyote
20 | title: kafka-backup
21 |
22 | - name: Setup Cluster to Backup
23 | entries:
24 | - name: Docker Compose Up
25 | command: docker-compose up -d
26 | - name: Clean previous data
27 | command: docker run -v /tmp/kafka-backup/:/kafka-backup/ kafka-backup-dev:latest rm -rf "/kafka-backup/001_simple_1partition_test/"
28 | - name: Wait for Kafka to get up
29 | command: docker logs to-backup-kafka 2>&1 | grep -q '\[KafkaServer id=1\] started'
30 | timeout: 30s
31 |
32 | - name: Create Topic for tests
33 | entries:
34 | - command: docker-compose exec -T to-backup-kafka bash -c '
35 | utils.py create_topic --topic backup-test-1partition --partitions 1 &&
36 | utils.py create_topic --topic backup-test-weird-msgs --partitions 1 &&
37 | utils.py create_topic --topic backup-test-empty-topic --partitions 3'
38 | - name: Produce Messages
39 | entries:
40 | - name: Produce 300 messages
41 | command: docker-compose exec -T to-backup-kafka bash -c '
42 | utils.py produce_messages --topic backup-test-1partition --partition 0 --start_num 0 --count 300 &&
43 | utils.py produce_messages --topic backup-test-empty-topic --partition 0 --start_num 0 --count 300'
44 | - name: Produce 'weird' messages
45 | command: docker-compose exec -T to-backup-kafka bash -c '
46 | utils.py produce_weird_messages --partition 0 --topic backup-test-weird-msgs'
47 | - name: Consume messages
48 | entries:
49 | - name: Consume 100 messages with cg-100
50 | command: docker-compose exec -T to-backup-kafka
51 | utils.py consume_messages --topic backup-test-1partition --consumer_group cg-100 --count 100
52 | - name: Consume 200 messages with cg-200
53 | command: docker-compose exec -T to-backup-kafka
54 | utils.py consume_messages --topic backup-test-1partition --consumer_group cg-200 --count 200
55 | - name: Consume 300 messages with cg-300
56 | command: docker-compose exec -T to-backup-kafka
57 | utils.py consume_messages --topic backup-test-1partition --consumer_group cg-300 --count 300
58 | - name: Check Consumer Group Offsets
59 | entries:
60 | - name: Count Messages
61 | command: docker-compose exec -T to-backup-kafka
62 | utils.py count_messages
63 | stdout_has:
64 | - "backup-test-1partition 0: 300"
65 | - "backup-test-empty-topic 0: 300"
66 | - "backup-test-empty-topic 1: 0"
67 | - "backup-test-empty-topic 2: 0"
68 | - name: Check Consumer Group cg-100
69 | command: docker-compose exec -T to-backup-kafka
70 | kafka-consumer-groups.sh --bootstrap-server localhost:9092 --describe --group cg-100
71 | stdout_has: [ 'backup-test-1partition 0 100 300 200' ]
72 | - name: Check Consumer Group cg-200
73 | command: docker-compose exec -T to-backup-kafka
74 | kafka-consumer-groups.sh --bootstrap-server localhost:9092 --describe --group cg-200
75 | stdout_has: [ 'backup-test-1partition 0 200 300 100' ]
76 | - name: Check Consumer Group cg-200
77 | command: docker-compose exec -T to-backup-kafka
78 | kafka-consumer-groups.sh --bootstrap-server localhost:9092 --describe --group cg-300
79 | stdout_has: [ 'backup-test-1partition 0 300 300 0' ]
80 |
81 | - name: Start Kafka Backup
82 | entries:
83 | - name: Start Kafka Backup
84 | command: >
85 | docker run -d -v /tmp/kafka-backup/:/kafka-backup/ --net=system_test_to-backup -p 18083:8083 --name to-backup --rm
86 | kafka-backup-dev:latest backup-standalone.sh --bootstrap-server to-backup-kafka:9092
87 | --target-dir /kafka-backup/001_simple_1partition_test/ --topics-regex 'backup-test.*'
88 | - command: sleep 30
89 | nolog: true
90 | - name: Check For errors
91 | timeout: 300s
92 | command: docker exec to-backup curl -vs "http://localhost:8083/connectors/backup-sink/status"
93 | stderr_has: ["200 OK"]
94 | stdout_has: ["RUNNING"]
95 | stdout_not_has: ["FAILED"]
96 |
97 | - name: Stop Cluster that was backed up
98 | entries:
99 | - name: Stop Kafka Backup
100 | command: docker kill to-backup
101 | - name: Docker Compose Down
102 | command: docker-compose stop to-backup-kafka
103 |
104 | - name: Restore
105 | entries:
106 | - name: Create Topic
107 | command: docker-compose exec -T restore-to-kafka bash -c '
108 | utils.py create_topic --topic backup-test-1partition --partitions 1 &&
109 | utils.py create_topic --topic backup-test-weird-msgs --partitions 1 &&
110 | utils.py create_topic --topic backup-test-empty-topic --partitions 3'
111 | - name: Run Kafka Restore
112 | command: >
113 | docker run -v /tmp/kafka-backup/:/kafka-backup/ --net=system_test_restore-to --name restore-to --rm
114 | kafka-backup-dev:latest restore-standalone.sh --bootstrap-server restore-to-kafka:9092
115 | --source-dir /kafka-backup/001_simple_1partition_test/ --topics 'backup-test-1partition,backup-test-empty-topic,backup-test-weird-msgs'
116 | timeout: 60s
117 | stdout_has: ['All records read.']
118 |
119 | - name: Verify Backup
120 | entries:
121 | - name: Verify Records
122 | timeout: 30s
123 | command: docker-compose exec -T restore-to-kafka bash -c '
124 | utils.py consume_verify_messages --topic backup-test-1partition --partition 0 --count 300 &&
125 | utils.py consume_verify_messages --topic backup-test-empty-topic --partition 0 --count 300'
126 | - name: Verify Weird Records
127 | timeout: 15s
128 | command: docker-compose exec -T restore-to-kafka bash -c '
129 | utils.py consume_verify_weird_messages --partition 0 --topic backup-test-weird-msgs'
130 | - name: Count Messages
131 | timeout: 30s
132 | command: docker-compose exec -T restore-to-kafka
133 | utils.py count_messages
134 | stdout_has:
135 | - "backup-test-1partition 0: 300"
136 | - "backup-test-empty-topic 0: 300"
137 | - "backup-test-empty-topic 1: 0"
138 | - "backup-test-empty-topic 2: 0"
139 | - name: Check Consumer Group cg-100
140 | timeout: 30s
141 | command: docker-compose exec -T restore-to-kafka
142 | kafka-consumer-groups.sh --bootstrap-server localhost:9092 --describe --group cg-100
143 | stdout_has: [ 'backup-test-1partition 0 100' ]
144 | - name: Check Consumer Group cg-200
145 | timeout: 30s
146 | command: docker-compose exec -T restore-to-kafka
147 | kafka-consumer-groups.sh --bootstrap-server localhost:9092 --describe --group cg-200
148 | stdout_has: [ 'backup-test-1partition 0 200' ]
149 | - name: Check Consumer Group cg-200
150 | timeout: 30s
151 | command: docker-compose exec -T restore-to-kafka
152 | kafka-consumer-groups.sh --bootstrap-server localhost:9092 --describe --group cg-300
153 | stdout_has: [ 'backup-test-1partition 0 300' ]
154 |
155 | - name: Clean-up Containers
156 | entries:
157 | - name: Docker Compose Down
158 | command: docker-compose down
159 | timeout: 15s
160 |
--------------------------------------------------------------------------------
/system_test/04_delete_old_segments.yaml:
--------------------------------------------------------------------------------
1 | # * Create a backup with multiple segments
2 | # * Delete some old segments
3 | # * Delete all indexes
4 | # * Recreate all indexes
5 | # * Do a restore
6 | - name: coyote
7 | title: kafka-backup
8 |
9 | - name: Setup Cluster to Backup
10 | entries:
11 | - name: Docker Compose Up
12 | command: docker-compose up -d
13 | - name: Clean previous data
14 | command: docker run -v /tmp/kafka-backup/:/kafka-backup/ kafka-backup-dev:latest rm -rf "/kafka-backup/04_delete_old_segment/"
15 | - name: Wait for Kafka to get up
16 | command: docker logs to-backup-kafka 2>&1 | grep -q '\[KafkaServer id=1\] started'
17 | timeout: 30s
18 |
19 | - name: Create Topic for tests
20 | entries:
21 | - command: docker-compose exec -T to-backup-kafka
22 | utils.py create_topic --topic backup-test-1partition --partitions 1
23 | - name: Produce Messages
24 | entries:
25 | - name: Produce 3000 messages
26 | command: docker-compose exec -T to-backup-kafka
27 | utils.py produce_messages --topic backup-test-1partition --partition 0 --start_num 0 --count 3000
28 | - name: Count Messages
29 | command: docker-compose exec -T to-backup-kafka
30 | utils.py count_messages
31 | stdout_has:
32 | - "backup-test-1partition 0: 3000"
33 |
34 | - name: Start Kafka Backup
35 | entries:
36 | - name: Start Kafka Backup
37 | command: >
38 | docker run -d -v /tmp/kafka-backup/:/kafka-backup/ --net=system_test_to-backup --name to-backup --rm
39 | kafka-backup-dev:latest backup-standalone.sh --bootstrap-server to-backup-kafka:9092
40 | --target-dir /kafka-backup/04_delete_old_segment/ --topics-regex 'backup-test.*' --max-segment-size 10485760
41 | - command: sleep 30
42 | nolog: true
43 |
44 | - name: Stop Cluster that was backed up
45 | entries:
46 | - name: Stop Kafka Backup
47 | command: docker kill to-backup
48 | - name: Docker Compose Down
49 | command: docker-compose stop to-backup-kafka
50 |
51 | - name: Delete old segment and restore the index
52 | entries:
53 | - name: Delete all indexes
54 | command: docker run -v /tmp/kafka-backup/:/kafka-backup/ kafka-backup-dev:latest bash -c \
55 | 'rm /kafka-backup/04_delete_old_segment/backup-test-1partition/*index*'
56 | - name: Delete old segment
57 | command: docker run -v /tmp/kafka-backup/:/kafka-backup/ kafka-backup-dev:latest bash -c \
58 | 'rm /kafka-backup/04_delete_old_segment/backup-test-1partition/segment_partition_000_from_offset_0000000000_records'
59 | - name: Restore segment and partition indexes
60 | command: >
61 | docker run -v /tmp/kafka-backup/:/kafka-backup/ kafka-backup-dev:latest bash -c '
62 | export TOPICDIR="/kafka-backup/04_delete_old_segment/backup-test-1partition/" &&
63 | export CLASSPATH="/connect-plugins/kafka-backup.jar" &&
64 | for f in "$TOPICDIR"/segment_partition_*_records ; do
65 | segment-index.sh --restore-index \
66 | --segment $f
67 | done &&
68 | partition-index.sh --restore --partition 0 --topic-dir "$TOPICDIR"'
69 |
70 | - name: Restore
71 | entries:
72 | - name: Create Topic
73 | command: docker-compose exec -T restore-to-kafka
74 | utils.py create_topic --topic backup-test-1partition --partitions 1
75 | - name: Run Kafka Restore
76 | command: >
77 | docker run -v /tmp/kafka-backup/:/kafka-backup/ --net=system_test_restore-to --name restore-to --rm
78 | kafka-backup-dev:latest restore-standalone.sh --bootstrap-server restore-to-kafka:9092
79 | --source-dir /kafka-backup/04_delete_old_segment/
80 | --topics 'backup-test-1partition'
81 | timeout: 60s
82 | stdout_has: ['All records read.']
83 |
84 | - name: Verify Backup
85 | entries:
86 | - name: Count Messages
87 | command: docker-compose exec -T restore-to-kafka
88 | utils.py count_messages
89 | stdout_has:
90 | - "backup-test-1partition 0: 1959"
91 | - name: Verify Records
92 | command: docker-compose exec -T restore-to-kafka
93 | utils.py consume_verify_messages --topic backup-test-1partition --partition 0 --start_num 1041 --count 1959
94 |
95 | - name: Clean-up Containers
96 | entries:
97 | - name: Docker Compose Down
98 | command: docker-compose down
99 |
--------------------------------------------------------------------------------
/system_test/README.md:
--------------------------------------------------------------------------------
1 | ## Usage
2 |
3 | Install coyote
4 |
5 | ```sh
6 | go get github.com/landoop/coyote
7 | ```
8 |
9 | Build Kafka Backup (from the root directory):
10 |
11 | ```sh
12 | ./gradlew shadowJar
13 | ```
14 |
15 | Then, just run coyote inside this directory.
16 |
17 | ```
18 | coyote
19 | ```
20 |
21 | When finished, open `coyote.html`.
22 |
23 | ## Software
24 |
25 | You need these programs to run the test:
26 | - [Coyote](https://github.com/Landoop/coyote/releases)
27 | - [Docker](https://docs.docker.com/engine/installation/)
28 | - [Docker Compose](https://docs.docker.com/engine/installation/)
29 |
30 | Everything else is set up automatically inside containers.
--------------------------------------------------------------------------------
/system_test/archive/001_simple_1partition_test/connect-backup-sink.properties:
--------------------------------------------------------------------------------
1 | name=backup-sink
2 | connector.class=de.azapps.kafkabackup.sink.BackupSinkConnector
3 | tasks.max=1
4 | topics.regex=backup-test.*
5 | key.converter=org.apache.kafka.connect.converters.ByteArrayConverter
6 | value.converter=org.apache.kafka.connect.converters.ByteArrayConverter
7 | header.converter=org.apache.kafka.connect.converters.ByteArrayConverter
8 | target.dir=/tmp/kafka-backup/001_simple_1partition_test/
9 | # 10MiB
10 | max.segment.size.bytes=10485760
11 | cluster.bootstrap.servers=localhost:9092
--------------------------------------------------------------------------------
/system_test/archive/001_simple_1partition_test/connect-backup-source.properties:
--------------------------------------------------------------------------------
1 | name=backup-source
2 | connector.class=de.azapps.kafkabackup.source.BackupSourceConnector
3 | tasks.max=1
4 | topics=backup-test-1partition,backup-test-empty
5 | key.converter=org.apache.kafka.connect.converters.ByteArrayConverter
6 | value.converter=org.apache.kafka.connect.converters.ByteArrayConverter
7 | header.converter=org.apache.kafka.connect.converters.ByteArrayConverter
8 | source.dir=/tmp/kafka-backup/001_simple_1partition_test/
9 | batch.size=1000
10 | cluster.bootstrap.servers=localhost:9092
11 | cluster.key.deserializer=org.apache.kafka.common.serialization.ByteArrayDeserializer
12 | cluster.value.deserializer=org.apache.kafka.common.serialization.ByteArrayDeserializer
--------------------------------------------------------------------------------
/system_test/archive/001_simple_1partition_test/test.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | SCRIPT_DIR=$(dirname $0)
3 | DATADIR=/tmp/kafka-backup/001_simple_1partition_test/
4 | source $SCRIPT_DIR/../utils.sh
5 |
6 | # Uses confluent cli
7 | #
8 | # * Stop kafka if its running
9 | kafka_stop
10 | # * Delete all data
11 | kafka_delete_data
12 | rm -rf $DATADIR
13 | mkdir -p $DATADIR
14 |
15 |
16 | ########################## Generate Data
17 | # * Start Kafka
18 | kafka_start
19 | # * Configure following topics:
20 | # * backup-test-1partition
21 | create_topic backup-test-1partition 1
22 | produce_messages backup-test-1partition 0 0 300
23 | consume_messages backup-test-1partition cg-100 100
24 | consume_messages backup-test-1partition cg-200 200
25 | consume_messages backup-test-1partition cg-300 300
26 |
27 | # 1 empty partition, one full
28 | create_topic backup-test-empty 3
29 | produce_messages backup-test-empty 0 0 300
30 |
31 |
32 |
33 | ########################## Backup
34 | # * Start Kafka Connect distributed
35 | kafka_connect_start
36 | # * Configure backup-sink:
37 | # * segment size: 10MiB
38 | # * topics.regex: backup-test-1partition
39 | kafka_connect_load_connector 001_simple_1partition_test_sink "$SCRIPT_DIR/connect-backup-sink.properties"
40 | # * Wait a few minutes
41 | sleep $((60*5))
42 |
43 |
44 | ########################## Destroy & Restore Cluster
45 | # * Stop Kafka
46 | kafka_stop
47 | # * Delete all data
48 | kafka_delete_data
49 | # * Start Kafka
50 | kafka_start
51 | # * Create all 3 topics as above (we are not testing zookeeper backup!)
52 | create_topic backup-test-1partition 1
53 | create_topic backup-test-empty 3
54 |
55 |
56 | ########################## Restore topic
57 | # * Start Kafka Connect distributed
58 | kafka_connect_start
59 | # * Configure backup-source
60 | kafka_connect_load_connector 001_simple_1partition_test_source $SCRIPT_DIR/connect-backup-source.properties
61 | kafka_connect_unload_connector 001_simple_1partition_test_source
62 |
63 | consume_verify_messages backup-test-1partition 0 300
64 |
--------------------------------------------------------------------------------
/system_test/archive/backup_with_burry/connect-backup-sink.properties:
--------------------------------------------------------------------------------
1 | name=backup-sink
2 | connector.class=de.azapps.kafkabackup.sink.BackupSinkConnector
3 | tasks.max=1
4 | topics.regex=backup-test.*
5 | key.converter=org.apache.kafka.connect.converters.ByteArrayConverter
6 | value.converter=org.apache.kafka.connect.converters.ByteArrayConverter
7 | header.converter=org.apache.kafka.connect.converters.ByteArrayConverter
8 | target.dir=/tmp/kafka-backup/backup_with_burry/topics
9 | # 10MiB
10 | max.segment.size.bytes=10485760
11 | cluster.bootstrap.servers=localhost:9092
--------------------------------------------------------------------------------
/system_test/archive/backup_with_burry/connect-backup-source.properties:
--------------------------------------------------------------------------------
1 | name=backup-source
2 | connector.class=de.azapps.kafkabackup.source.BackupSourceConnector
3 | tasks.max=1
4 | topics=backup-test-1partition
5 | key.converter=org.apache.kafka.connect.converters.ByteArrayConverter
6 | value.converter=org.apache.kafka.connect.converters.ByteArrayConverter
7 | header.converter=org.apache.kafka.connect.converters.ByteArrayConverter
8 | source.dir=/tmp/kafka-backup/backup_with_burry/topics
9 | batch.size=1000
10 | cluster.bootstrap.servers=localhost:9092
11 | cluster.key.deserializer=org.apache.kafka.common.serialization.ByteArrayDeserializer
12 | cluster.value.deserializer=org.apache.kafka.common.serialization.ByteArrayDeserializer
--------------------------------------------------------------------------------
/system_test/archive/backup_with_burry/test.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | SCRIPT_DIR=$(dirname $0)
3 | DATADIR=/tmp/kafka-backup/backup_with_burry
4 | source $SCRIPT_DIR/../utils.sh
5 |
6 | # Uses confluent cli
7 | #
8 | # * Stop kafka if its running
9 | kafka_stop
10 | # * Delete all data
11 | kafka_delete_data
12 | rm -rf $DATADIR
13 | mkdir -p $DATADIR/{burry,topics}
14 |
15 |
16 | ########################## Generate Data
17 | # * Start Kafka
18 | kafka_start
19 | # * Configure following topics:
20 | # * backup-test-1partition
21 | create_topic backup-test-1partition 1
22 | produce_messages backup-test-1partition 0 0 300
23 | consume_messages backup-test-1partition cg-100 100
24 | consume_messages backup-test-1partition cg-200 200
25 | consume_messages backup-test-1partition cg-300 300
26 |
27 |
28 | ########################## Backup
29 | # * Start Kafka Connect distributed
30 | kafka_connect_start
31 | # * Configure backup-sink:
32 | # * segment size: 10MiB
33 | # * topics.regex: backup-test-1partition
34 | kafka_connect_load_connector 001_simple_1partition_test_sink "$SCRIPT_DIR/connect-backup-sink.properties"
35 | # * Wait a few minutes
36 | sleep $((60*5))
37 |
38 | ########################## Backup Zookeeper
39 | burry_backup $DATADIR/burry
40 |
41 | ########################## Destroy & Restore Cluster
42 | # * Stop Kafka
43 | kafka_stop
44 | # * Delete all data
45 | kafka_delete_data
46 | # * Start Kafka
47 | kafka_start
48 |
49 | ########################## Restore Zookeeper
50 | burry_restore $DATADIR/burry
51 | # Restart Kafka
52 | kafka_stop
53 | kafka_start
54 |
55 |
56 | ########################## Restore topic
57 | # * Start Kafka Connect distributed
58 | kafka_connect_start
59 | # * Configure backup-source
60 | kafka_connect_load_connector 001_simple_1partition_test_source $SCRIPT_DIR/connect-backup-source.properties
61 | sleep $((60*5))
62 | kafka_connect_unload_connector 001_simple_1partition_test_source
63 |
64 | consume_verify_messages backup-test-1partition 0 300
65 |
--------------------------------------------------------------------------------
/system_test/archive/full_test/connect-backup-sink.properties:
--------------------------------------------------------------------------------
1 | name=backup-sink
2 | connector.class=de.azapps.kafkabackup.sink.BackupSinkConnector
3 | tasks.max=1
4 | topics.regex=backup-test.*
5 | key.converter=org.apache.kafka.connect.converters.ByteArrayConverter
6 | value.converter=org.apache.kafka.connect.converters.ByteArrayConverter
7 | header.converter=org.apache.kafka.connect.converters.ByteArrayConverter
8 | target.dir=/tmp/kafka-backup/full_test
9 | # 10MiB
10 | max.segment.size.bytes=10485760
11 | cluster.bootstrap.servers=localhost:9092
--------------------------------------------------------------------------------
/system_test/archive/full_test/connect-backup-source.properties:
--------------------------------------------------------------------------------
1 | name=backup-source
2 | connector.class=de.azapps.kafkabackup.source.BackupSourceConnector
3 | tasks.max=1
4 | topics=backup-test-1partition,backup-test-3partitions,backup-test-10partitions
5 | key.converter=org.apache.kafka.connect.converters.ByteArrayConverter
6 | value.converter=org.apache.kafka.connect.converters.ByteArrayConverter
7 | header.converter=org.apache.kafka.connect.converters.ByteArrayConverter
8 | source.dir=/tmp/kafka-backup/full_test/
9 | batch.size=1000
10 | cluster.bootstrap.servers=localhost:9092
11 | cluster.key.deserializer=org.apache.kafka.common.serialization.ByteArrayDeserializer
12 | cluster.value.deserializer=org.apache.kafka.common.serialization.ByteArrayDeserializer
--------------------------------------------------------------------------------
/system_test/archive/full_test/round_trip.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | SCRIPT_DIR=$(dirname $0)
3 | DATADIR=/tmp/kafka-backup/full_test/
4 | source $SCRIPT_DIR/../utils.sh
5 | NUM_MSG=100
6 |
7 | # Uses confluent cli
8 | #
9 | # * Stop kafka if its running
10 | kafka_stop
11 | # * Delete all data
12 | kafka_delete_data
13 | rm -rf $DATADIR
14 | mkdir -p $DATADIR
15 | # * Start Kafka
16 | kafka_start
17 | # * Configure following topics:
18 | # * backup-test-1partition
19 | # * backup-test-3partitions
20 | create_topic backup-test-1partition 1
21 | create_topic backup-test-3partitions 3
22 | # * Produce 3,00 messages, 10KiB each to each partition
23 | #
24 | # We need to chunk the production of messages as otherwise we cannot
25 | # guarantee that the group consumer will evenly consume the partitions.
26 | produce_messages backup-test-1partition 0 0 $((3 * NUM_MSG))
27 | # backup-test-3partition
28 | produce_messages backup-test-3partitions 0 0 $((3 * NUM_MSG))
29 | produce_messages backup-test-3partitions 1 0 $((3 * NUM_MSG))
30 | produce_messages backup-test-3partitions 2 0 $((3 * NUM_MSG))
31 | # * Consume all messages with consumer-group `cg-3k`
32 | consume_messages backup-test-1partition cg-3k $((3 * NUM_MSG))
33 | consume_messages backup-test-3partitions cg-3k $((9 * NUM_MSG))
34 | kafka_group_describe cg-3k
35 | # * Produce 2 * NUM_MSG messages
36 | produce_messages backup-test-1partition 0 $((3 * NUM_MSG)) $((2 * NUM_MSG))
37 | # backup-test-3partition
38 | produce_messages backup-test-3partitions 0 $((3 * NUM_MSG)) $((2 * NUM_MSG))
39 | produce_messages backup-test-3partitions 1 $((3 * NUM_MSG)) $((2 * NUM_MSG))
40 | produce_messages backup-test-3partitions 2 $((3 * NUM_MSG)) $((2 * NUM_MSG))
41 | # * Consume all messages with consumer-group `cg-5k`
42 | consume_messages backup-test-1partition cg-5k $((5 * NUM_MSG))
43 | consume_messages backup-test-3partitions cg-5k $((15 * NUM_MSG))
44 | # * Produce 100 more messages
45 | produce_messages backup-test-1partition 0 $((5 * NUM_MSG)) $((1 * NUM_MSG))
46 | # backup-test-3partition
47 | produce_messages backup-test-3partitions 0 $((5 * NUM_MSG)) $((1 * NUM_MSG))
48 | produce_messages backup-test-3partitions 1 $((5 * NUM_MSG)) $((1 * NUM_MSG))
49 | produce_messages backup-test-3partitions 2 $((5 * NUM_MSG)) $((1 * NUM_MSG))
50 | # * Start Kafka Connect distributed
51 | kafka_connect_start
52 | # * Configure backup-sink:
53 | # * segment size: 10MiB
54 | # * topics.regex: backup-test-*
55 | sleep 10
56 | kafka_connect_load_connector backup-sink "$SCRIPT_DIR/connect-backup-sink.properties"
57 | sleep 10
58 | # * Create another topic:
59 | # * backup-test-10partitions
60 |
61 | create_topic backup-test-10partitions 10
62 | # * Produce 1,00 messages as above and consume 500 messages as above
63 | for i in {0..9} ; do
64 | produce_messages backup-test-10partitions $i 0 $((5 * NUM_MSG))
65 | done
66 | # To force segmentation rolling
67 | produce_messages backup-test-1partition 0 $((6 * NUM_MSG)) $((15 * NUM_MSG))
68 | # Consume some messages
69 | # * Wait a few minutes
70 | sleep $((60*5))
71 | # * Stop Kafka
72 | kafka_stop
73 | # * Delete all data
74 | kafka_delete_data
75 | # * Start Kafka
76 | kafka_start
77 | # * Create all 3 topics as above (we are not testing zookeeper backup!)
78 | create_topic backup-test-1partition 1
79 | create_topic backup-test-3partitions 3
80 | create_topic backup-test-10partitions 10
81 | # * Start Kafka Connect distributed
82 | kafka_connect_start
83 | # * Configure backup-source
84 | kafka_connect_load_connector backup-source $SCRIPT_DIR/connect-backup-source.properties
85 | # * Wait for restore to finish
86 | sleep $((60*15))
87 | kafka_connect_unload_connector backup-source
88 | # * Read all messages and check that they are the same as the ones that were written.
89 | # * Subscribe to Kafka using the consumer groups as above (`cg-5k` and `cg-3k`) and check whether they are at the correct position
90 |
91 | consume_verify_messages backup-test-1partition 0 $((21 * NUM_MSG))
92 | for i in {0..2} ; do
93 | consume_verify_messages backup-test-3partitions $i $((6 * NUM_MSG))
94 | done
95 | for i in {0..9} ; do
96 | consume_verify_messages backup-test-10partitions $i $((5 * NUM_MSG))
97 | done
98 |
--------------------------------------------------------------------------------
/system_test/archive/utils.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | kafka_start() {
4 | confluent local start kafka
5 | }
6 |
7 | kafka_stop() {
8 | confluent local stop
9 | }
10 |
11 | kafka_delete_data() {
12 | rm -r /tmp/confluent.*
13 | }
14 |
15 | kafka-topics() {
16 | "$CONFLUENT_HOME/bin/kafka-topics" $@
17 | }
18 | kafka-console-consumer() {
19 | "$CONFLUENT_HOME/bin/kafka-console-consumer" $@
20 | }
21 | kafka-consumer-groups() {
22 | "$CONFLUENT_HOME/bin/kafka-consumer-groups" $@
23 | }
24 |
25 | create_topic() {
26 | TOPIC=$1
27 | PARTITIONS=$2
28 | if [ -z "$PARTITIONS" ]; then
29 | echo "USAGE: $0 [TOPIC] [PARTITIONS]"
30 | return 255
31 | fi
32 |
33 | kafka-topics --create --bootstrap-server localhost:9092 --topic "$TOPIC" --partitions "$PARTITIONS" --replication-factor 1
34 | }
35 |
36 | gen_message() {
37 | PARTITION=$1
38 | NUM=$2
39 | if [ -z "$NUM" ]; then
40 | echo "USAGE: $0 [PARTITION] [NUM] (SIZE)"
41 | return 255
42 | fi
43 | SIZE=$3
44 | if [ -z "$SIZE" ]; then
45 | SIZE=7500 # 10k Bytes base64
46 | fi
47 | VALUE=$(dd if=/dev/urandom bs=$SIZE count=1 2>/dev/null | base64 -w0)
48 | CHECKSUM=$(echo "$VALUE" | md5sum | cut -d' ' -f1)
49 | KEY="part_${PARTITION}_num_${NUM}_${CHECKSUM}"
50 | echo "${KEY},${VALUE}"
51 | }
52 |
53 | gen_messages() {
54 | PARTITION=$1
55 | START_NUM=$2
56 | COUNT=$3
57 | if [ -z "$COUNT" ]; then
58 | echo "USAGE: $0 [PARTITION] [START_NUM] [COUNT] (SIZE)"
59 | return 255
60 | fi
61 | SIZE=$4
62 | for NUM in {$START_NUM..$((START_NUM + COUNT - 1))}; do
63 | if [ "0" -eq "$(((NUM - START_NUM) % 100))" ]; then
64 | echo -e -n "\rProduced $((NUM - START_NUM))/$COUNT messages" >/dev/stderr
65 | fi
66 | gen_message "$PARTITION" $NUM "$SIZE"
67 | done
68 | echo ""
69 | }
70 |
71 | produce_messages() {
72 | TOPIC=$1
73 | PARTITION=$2
74 | START_NUM=$3
75 | COUNT=$4
76 | if [ -z "$COUNT" ]; then
77 | echo "USAGE: $0 [TOPIC] [PARTITION] [START_NUM] [COUNT] (SIZE)"
78 | return 255
79 | fi
80 | SIZE=$5
81 |
82 | gen_messages "$PARTITION" "$START_NUM" "$COUNT" "$SIZE" | kafkacat -P -b localhost:9092 -t "$TOPIC" -p "$PARTITION" -K ","
83 | }
84 |
85 | verify_messages() {
86 | PREVIOUS_NUM="-1"
87 | while read -r MESSAGE; do
88 | if [ "0" -eq "$(((PREVIOUS_NUM + 1) % 10))" ]; then
89 | echo -e -n "\rVerified $((PREVIOUS_NUM + 1)) messages" >/dev/stderr
90 | fi
91 | KEY=$(echo "$MESSAGE" | awk '{print $1}')
92 | KEY_MATCH=$(echo "$KEY" | sed 's/part_\([0-9]*\)_num_\([0-9]*\)_\(.*\)$/\1\t\2\t\3/')
93 | KEY_PARTITION=$(echo "$KEY_MATCH" | awk '{print $1}')
94 | KEY_NUM=$(echo "$KEY_MATCH" | awk '{print $2}')
95 | KEY_CHECKSUM=$(echo "$KEY_MATCH" | awk '{print $3}')
96 |
97 | VALUE=$(echo "$MESSAGE" | awk '{print $2}')
98 | VALUE_CHECKSUM=$(echo "$VALUE" | md5sum | cut -d' ' -f1)
99 |
100 | if [ ! "$KEY_NUM" -eq "$((PREVIOUS_NUM + 1))" ]; then
101 | echo "Missing message. Previous message has num $PREVIOUS_NUM. This message has num $KEY_NUM"
102 | return 255
103 | fi
104 | PREVIOUS_NUM=$KEY_NUM
105 |
106 | if [ "$KEY_CHECKSUM" != "$VALUE_CHECKSUM" ]; then
107 | echo "Partition $KEY_PARTITION, Key $KEY_NUM, KChk $KEY_CHECKSUM, vlength ${#VALUE}, vchk: $VALUE_CHECKSUM"
108 |
109 | echo "Checksum mismatch: Checksum in key ($KEY_CHECKSUM) does not match Checksum of value ($VALUE_CHECKSUM)"
110 | return 255
111 | fi
112 | done
113 | echo -e "\rVerified $((PREVIOUS_NUM + 1)) messages"
114 | }
115 |
116 | consume_verify_messages() {
117 | TOPIC=$1
118 | PARTITION=$2
119 | COUNT=$3
120 | if [ -z "$COUNT" ]; then
121 | echo "USAGE: $0 [TOPIC] [PARTITION] [COUNT]"
122 | return 255
123 | fi
124 |
125 | kafka-console-consumer \
126 | --bootstrap-server localhost:9092 \
127 | --from-beginning --property print.key=true \
128 | --topic "$TOPIC" \
129 | --max-messages="$COUNT" \
130 | --partition="$PARTITION" 2>&/dev/null |
131 | verify_messages
132 | }
133 |
134 | consume_messages() {
135 | TOPIC=$1
136 | CONSUMER_GROUP=$2
137 | COUNT=$3
138 | if [ -z "$COUNT" ]; then
139 | echo "USAGE: $0 [TOPIC] [CONSUMER GROUP] [COUNT]"
140 | return 255
141 | fi
142 |
143 | MESSAGES=$(kafka-console-consumer \
144 | --bootstrap-server localhost:9092 \
145 | --from-beginning --property print.key=true \
146 | --topic "$TOPIC" \
147 | --max-messages "$COUNT" \
148 | --group "$CONSUMER_GROUP") # 2>/dev/null)
149 | echo "Consumed $(echo "$MESSAGES" | wc -l) messages"
150 | }
151 |
152 | kafka_connect_start() {
153 | confluent local start connect
154 | }
155 |
156 | kafka_connect_load_connector() {
157 | NAME=$1
158 | PROPS="$2"
159 | if [ -z "$PROPS" ]; then
160 | echo "USAGE: $0 [NAME] [PROPS FILE]"
161 | return 255
162 | fi
163 | cp "$PROPS" /tmp/connect.properties
164 | confluent local load "$NAME" -- -d /tmp/connect.properties
165 | rm /tmp/connect.properties
166 | }
167 |
168 | kafka_connect_unload_connector() {
169 | NAME=$1
170 | if [ -z "$NAME" ]; then
171 | echo "USAGE: $0 [NAME]"
172 | return 255
173 | fi
174 | confluent local unload "$NAME"
175 | }
176 |
177 | kafka_group_describe() {
178 | GROUP=$1
179 | if [ -z "$GROUP" ]; then
180 | echo "USAGE: $0 [GROUP]"
181 | return 255
182 | fi
183 | kafka-consumer-groups --bootstrap-server localhost:9092 --describe --group "$GROUP"
184 | }
185 |
186 | burry_backup() {
187 | TARGET_DIR=$1
188 | if [ -z "$TARGET_DIR" ]; then
189 | echo "USAGE: $0 [TARGET_DIR]"
190 | return 255
191 | fi
192 | docker run --network=host -v "$TARGET_DIR":/data azapps/burry -e localhost:2181 -t local
193 | }
194 |
195 | burry_restore() {
196 | SOURCE_DIR=$1
197 | if [ -z "$SOURCE_DIR" ]; then
198 | echo "USAGE: $0 [SOURCE_DIR]"
199 | return 255
200 | fi
201 | SNAPSHOT=$(ls "$DATADIR"/burry | tail -n 1 | sed 's/.zip//')
202 | docker run --network=host -v "$SOURCE_DIR":/data azapps/burry --operation=restore --snapshot="$SNAPSHOT" -e localhost:2181 -t local
203 | }
204 |
--------------------------------------------------------------------------------
/system_test/docker-compose.yml:
--------------------------------------------------------------------------------
1 | version: '3.3'
2 | services:
3 | # To Backup
4 | to-backup-zk:
5 | build:
6 | context: ./utils
7 | dockerfile: Dockerfile
8 | container_name: to-backup-zk
9 | hostname: to-backup-zk
10 | ports:
11 | - 12181:2181
12 | volumes:
13 | - "./utils/kafka-configs/zookeeper.properties:/etc/zookeeper.properties"
14 | command: "zookeeper-server-start.sh /etc/zookeeper.properties"
15 | networks:
16 | - to-backup
17 |
18 | to-backup-kafka:
19 | build:
20 | context: ./utils
21 | dockerfile: Dockerfile
22 | container_name: to-backup-kafka
23 | hostname: to-backup-kafka
24 | volumes:
25 | - "./utils/kafka-configs/to-backup-kafka.properties:/etc/kafka.properties"
26 | - "./utils/utils.py:/usr/bin/utils.py"
27 | ports:
28 | - 19092:19092
29 | networks:
30 | - to-backup
31 | depends_on:
32 | - to-backup-zk
33 | command: "kafka-server-start.sh /etc/kafka.properties"
34 |
35 | ################################################################################################
36 | # Restore to
37 |
38 | restore-to-zk:
39 | build:
40 | context: ./utils
41 | dockerfile: Dockerfile
42 | container_name: restore-to-zk-1
43 | hostname: restore-to-zk-1
44 | ports:
45 | - 22181:2181
46 | volumes:
47 | - "./utils/kafka-configs/zookeeper.properties:/etc/zookeeper.properties"
48 | command: "zookeeper-server-start.sh /etc/zookeeper.properties"
49 | networks:
50 | - restore-to
51 |
52 | restore-to-kafka:
53 | build:
54 | context: ./utils
55 | dockerfile: Dockerfile
56 | container_name: restore-to-kafka
57 | hostname: restore-to-kafka
58 | volumes:
59 | - "./utils/kafka-configs/restore-to-kafka.properties:/etc/kafka.properties"
60 | - "./utils/utils.py:/usr/bin/utils.py"
61 | ports:
62 | - 29092:29092
63 | networks:
64 | - restore-to
65 | depends_on:
66 | - restore-to-zk
67 | command: "kafka-server-start.sh /etc/kafka.properties"
68 |
69 | networks:
70 | to-backup:
71 | restore-to:
--------------------------------------------------------------------------------
/system_test/utils/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM kafka-backup-dev:latest
2 |
3 | RUN apk add --no-cache make gcc g++ cmake curl pkgconfig perl bsd-compat-headers zlib-dev lz4-dev openssl-dev \
4 | curl-dev libcurl lz4-libs ca-certificates python3 bash python3-dev
5 |
6 | # Build librdkafka
7 | RUN mkdir /usr/src && cd /usr/src/ && \
8 | curl https://codeload.github.com/edenhill/librdkafka/tar.gz/master | tar xzf - && \
9 | cd librdkafka-master && \
10 | ./configure && \
11 | make && make install && \
12 | cd / && rm -rf /usr/src/
13 |
14 | # Install confluent-kafka python
15 |
16 | RUN pip3 install confluent-kafka==1.3.0 pykafka==2.8.0dev1
17 | COPY utils.py /usr/bin/utils.py
--------------------------------------------------------------------------------
/system_test/utils/kafka-configs/restore-to-kafka.properties:
--------------------------------------------------------------------------------
1 | broker.id=0
2 | log.dirs=/tmp/kafka-logs
3 | zookeeper.connect=restore-to-zk:2181
4 | advertised.listeners=INDOCKER://restore-to-kafka:9092,OUTDOCKER://localhost:29092
5 | listeners=INDOCKER://:9092,OUTDOCKER://:29092
6 | listener.security.protocol.map=INDOCKER:PLAINTEXT,OUTDOCKER:PLAINTEXT
7 | inter.broker.listener.name=INDOCKER
8 | offsets.topic.replication.factor=1
9 | offsets.topic.num.partitions=1
10 | auto.create.topics.enable=false
--------------------------------------------------------------------------------
/system_test/utils/kafka-configs/to-backup-kafka.properties:
--------------------------------------------------------------------------------
1 | broker.id=0
2 | log.dirs=/tmp/kafka-logs
3 | zookeeper.connect=to-backup-zk:2181
4 | advertised.listeners=INDOCKER://to-backup-kafka:9092,OUTDOCKER://localhost:19092
5 | listeners=INDOCKER://:9092,OUTDOCKER://:19092
6 | listener.security.protocol.map=INDOCKER:PLAINTEXT,OUTDOCKER:PLAINTEXT
7 | inter.broker.listener.name=INDOCKER
8 | offsets.topic.replication.factor=1
9 | offsets.topic.num.partitions=1
10 | auto.create.topics.enable=false
--------------------------------------------------------------------------------
/system_test/utils/kafka-configs/zookeeper.properties:
--------------------------------------------------------------------------------
1 | dataDir=/tmp/zookeeper
2 | clientPort=2181
3 | maxClientCnxns=0
--------------------------------------------------------------------------------
/system_test/utils/runutil:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | source /root/utils.sh
3 | "$@"
--------------------------------------------------------------------------------
/system_test/utils/utils.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | create_topic() {
4 | TOPIC=$1
5 | PARTITIONS=$2
6 | if [ -z "$PARTITIONS" ] || [ -n "$3" ]; then
7 | echo "USAGE: $0 [TOPIC] [PARTITIONS]"
8 | return 255
9 | fi
10 |
11 | kafka-topics --create --bootstrap-server localhost:9092 --topic "$TOPIC" --partitions "$PARTITIONS" --replication-factor 1
12 | }
13 | export -f create_topic
14 |
15 | gen_message() {
16 | PARTITION=$1
17 | NUM=$2
18 | SIZE=$3
19 | if [ -z "$NUM" ] || [ -n "$4" ]; then
20 | echo "USAGE: $0 [PARTITION] [NUM] (SIZE)"
21 | return 255
22 | fi
23 | if [ -z "$SIZE" ]; then
24 | SIZE=7500 # 10k Bytes base64
25 | fi
26 | VALUE=$(dd if=/dev/urandom bs=$SIZE count=1 2>/dev/null | base64 -w0)
27 | CHECKSUM=$(echo "$VALUE" | md5sum | cut -d' ' -f1)
28 | KEY="part_${PARTITION}_num_${NUM}_${CHECKSUM}"
29 | echo "${KEY},${VALUE}"
30 | }
31 | export -f gen_message
32 |
33 | gen_messages() {
34 | PARTITION=$1
35 | START_NUM=$2
36 | COUNT=$3
37 | if [ -z "$COUNT" ] || [ -n "$4" ]; then
38 | echo "USAGE: $0 [PARTITION] [START_NUM] [COUNT] (SIZE)"
39 | return 255
40 | fi
41 | SIZE=$4
42 | for NUM in $(seq $START_NUM $((START_NUM + COUNT - 1))); do
43 | if [ "0" -eq "$(((NUM - START_NUM) % 100))" ]; then
44 | echo -e -n "\rProduced $((NUM - START_NUM))/$COUNT messages" >/dev/stderr
45 | fi
46 | gen_message "$PARTITION" $NUM "$SIZE"
47 | done
48 | echo ""
49 | }
50 | export -f gen_messages
51 |
52 | produce_messages() {
53 | TOPIC=$1
54 | PARTITION=$2
55 | START_NUM=$3
56 | COUNT=$4
57 | SIZE=$5
58 | if [ -z "$COUNT" ] || [ -n "$6" ]; then
59 | echo "USAGE: $0 [TOPIC] [PARTITION] [START_NUM] [COUNT] (SIZE)"
60 | return 255
61 | fi
62 |
63 | gen_messages "$PARTITION" "$START_NUM" "$COUNT" "$SIZE" | kafkacat -P -b localhost:9092 -t "$TOPIC" -p "$PARTITION" -K ","
64 | }
65 | export -f produce_messages
66 |
67 | verify_messages() {
68 | PREVIOUS_NUM="$1"
69 | if [ -z "$PREVIOUS_NUM" ]; then
70 | PREVIOUS_NUM="-1"
71 | fi
72 | while read -r MESSAGE; do
73 | if [ "0" -eq "$(((PREVIOUS_NUM + 1) % 100))" ]; then
74 | echo -e -n "\rVerified $((PREVIOUS_NUM + 1)) messages" >/dev/stderr
75 | fi
76 | KEY=$(echo "$MESSAGE" | awk '{print $1}')
77 | KEY_MATCH=$(echo "$KEY" | sed 's/part_\([0-9]*\)_num_\([0-9]*\)_\(.*\)$/\1\t\2\t\3/')
78 | KEY_PARTITION=$(echo "$KEY_MATCH" | awk '{print $1}')
79 | KEY_NUM=$(echo "$KEY_MATCH" | awk '{print $2}')
80 | KEY_CHECKSUM=$(echo "$KEY_MATCH" | awk '{print $3}')
81 |
82 | VALUE=$(echo "$MESSAGE" | awk '{print $2}')
83 | VALUE_CHECKSUM=$(echo "$VALUE" | md5sum | cut -d' ' -f1)
84 |
85 | if [ ! "$KEY_NUM" -eq "$((PREVIOUS_NUM + 1))" ]; then
86 | echo "Missing message. Previous message has num $PREVIOUS_NUM. This message has num $KEY_NUM"
87 | return 255
88 | fi
89 | PREVIOUS_NUM=$KEY_NUM
90 |
91 | if [ "$KEY_CHECKSUM" != "$VALUE_CHECKSUM" ]; then
92 | echo "Partition $KEY_PARTITION, Key $KEY_NUM, KChk $KEY_CHECKSUM, vlength ${#VALUE}, vchk: $VALUE_CHECKSUM"
93 |
94 | echo "Checksum mismatch: Checksum in key ($KEY_CHECKSUM) does not match Checksum of value ($VALUE_CHECKSUM)"
95 | return 255
96 | fi
97 | done
98 | echo -e "\rVerified $((PREVIOUS_NUM + 1)) messages"
99 | }
100 | export -f verify_messages
101 |
102 | consume_verify_messages() {
103 | TOPIC=$1
104 | PARTITION=$2
105 | START_NUM=$3
106 | COUNT=$4
107 | if [ -z "$COUNT" ]; then
108 | COUNT="$START_NUM"
109 | START_NUM="0"
110 | fi
111 |
112 | if [ -z "$COUNT" ] || [ -n "$5" ]; then
113 | echo "USAGE: $0 [TOPIC] [PARTITION] ([START_NUM]) [COUNT]"
114 | return 255
115 | fi
116 |
117 | kafka-console-consumer \
118 | --bootstrap-server localhost:9092 \
119 | --from-beginning --property print.key=true \
120 | --topic "$TOPIC" \
121 | --max-messages="$COUNT" \
122 | --partition="$PARTITION" 2>/dev/null |
123 | verify_messages $((START_NUM - 1))
124 | }
125 | export -f consume_verify_messages
126 |
127 | consume_messages() {
128 | TOPIC=$1
129 | CONSUMER_GROUP=$2
130 | COUNT=$3
131 | if [ -z "$COUNT" ] || [ -n "$4" ]; then
132 | echo "USAGE: $0 [TOPIC] [CONSUMER GROUP] [COUNT]"
133 | return 255
134 | fi
135 |
136 | MESSAGES=$(kafka-console-consumer \
137 | --bootstrap-server localhost:9092 \
138 | --from-beginning --property print.key=true \
139 | --topic "$TOPIC" \
140 | --max-messages "$COUNT" \
141 | --group "$CONSUMER_GROUP") # 2>/dev/null)
142 | echo "Consumed $(echo "$MESSAGES" | wc -l) messages"
143 | }
144 | export -f consume_messages
145 |
146 | kafka_group_describe() {
147 | GROUP=$1
148 | if [ -z "$GROUP" ] || [ -n "$2" ]; then
149 | echo "USAGE: $0 [GROUP]"
150 | return 255
151 | fi
152 | kafka-consumer-groups --bootstrap-server localhost:9092 --describe --group "$GROUP"
153 | }
154 | export -f kafka_group_describe
155 |
156 | burry_backup() {
157 | TARGET_DIR=$1
158 | if [ -z "$TARGET_DIR" ] || [ -n "$2" ]; then
159 | echo "USAGE: $0 [TARGET_DIR]"
160 | return 255
161 | fi
162 | docker run --network=host -v "$TARGET_DIR":/data azapps/burry -e localhost:2181 -t local
163 | }
164 | export -f burry_backup
165 |
166 | burry_restore() {
167 | SOURCE_DIR=$1
168 | if [ -z "$SOURCE_DIR" ] || [ -n "$2" ]; then
169 | echo "USAGE: $0 [SOURCE_DIR]"
170 | return 255
171 | fi
172 | SNAPSHOT=$(ls "$DATADIR"/burry | tail -n 1 | sed 's/.zip//')
173 | docker run --network=host -v "$SOURCE_DIR":/data azapps/burry --operation=restore --snapshot="$SNAPSHOT" -e localhost:2181 -t local
174 | }
175 | export -f burry_restore
176 |
--------------------------------------------------------------------------------