├── tools ├── .gitignore └── maven │ └── suppressions.xml ├── flink-statebackend-heap-spillable ├── src │ ├── main │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── flink │ │ │ └── runtime │ │ │ └── state │ │ │ └── heap │ │ │ ├── SpillAndLoadManager.java │ │ │ ├── space │ │ │ ├── Constants.java │ │ │ ├── AllocateStrategy.java │ │ │ ├── ChunkAllocator.java │ │ │ ├── DirectBufferChunkAllocator.java │ │ │ ├── HeapBufferChunkAllocator.java │ │ │ ├── SpaceConstants.java │ │ │ ├── BucketAllocator.java │ │ │ ├── AbstractChunk.java │ │ │ ├── Allocator.java │ │ │ ├── SpaceUtils.java │ │ │ ├── DefaultChunkImpl.java │ │ │ ├── Chunk.java │ │ │ ├── AbstractChunkAllocator.java │ │ │ ├── BlockAllocator.java │ │ │ ├── Bucket.java │ │ │ ├── DirectBucketAllocator.java │ │ │ ├── PowerTwoBucketAllocator.java │ │ │ └── SpaceAllocator.java │ │ │ ├── NodeStatus.java │ │ │ ├── estimate │ │ │ ├── StateMemoryEstimator.java │ │ │ ├── ValueStateMemoryEstimator.java │ │ │ ├── AbstractStateMemoryEstimator.java │ │ │ ├── SampleStateMemoryEstimator.java │ │ │ ├── StateMemoryEstimatorFactory.java │ │ │ ├── ListStateMemoryEstimator.java │ │ │ ├── Constants.java │ │ │ └── MapStateMemoryEstimator.java │ │ │ ├── LevelIndexHeader.java │ │ │ ├── SpillableStateBackendFactory.java │ │ │ ├── CheckpointManager.java │ │ │ ├── SpillableSnapshotStrategySynchronicityBehavior.java │ │ │ ├── SkipListValueSerializer.java │ │ │ ├── SpillableStateTableSnapshot.java │ │ │ ├── SkipListKeyComparator.java │ │ │ ├── CheckpointManagerImpl.java │ │ │ ├── SpillableValueState.java │ │ │ ├── SpillableStateTable.java │ │ │ ├── OnHeapLevelIndexHeader.java │ │ │ ├── SpillableFoldingState.java │ │ │ ├── SpillableReducingState.java │ │ │ ├── SpillableAggregatingState.java │ │ │ └── SpillableOptions.java │ └── test │ │ ├── resources │ │ └── log4j2-test.properties │ │ └── java │ │ └── org │ │ └── apache │ │ └── flink │ │ └── runtime │ │ └── state │ │ ├── ttl │ │ ├── SpillableTtlUtils.java │ │ └── SpillableTtlMapStateAllEntriesTestContext.java │ │ └── heap │ │ ├── TestHeapStatusMonitor.java │ │ ├── SpillableStateBackendMigrationTest.java │ │ ├── ttl │ │ └── SpillableSnapshotTtlStateTest.java │ │ ├── SpillableOptionsTest.java │ │ ├── space │ │ └── MmapChunkAllocatorTest.java │ │ ├── SkipListSerializerTest.java │ │ ├── SpillableStateBackendTest.java │ │ ├── TestAllocator.java │ │ ├── OnHeapLevelIndexHeaderTest.java │ │ ├── SnapshotCompatibilityTest.java │ │ └── SpillableStateBackendFactoryTest.java └── pom.xml ├── .gitignore ├── flink-spillable-benchmark ├── README.md ├── pom.xml └── src │ └── main │ └── java │ └── org │ └── apache │ └── flink │ └── spillable │ └── benchmark │ ├── JobConfig.java │ ├── ThrottledIterator.java │ ├── WordSource.java │ └── WordCount.java ├── pom.xml └── README.md /tools/.gitignore: -------------------------------------------------------------------------------- 1 | _qa_workdir 2 | merge_pull_request.sh 3 | artifacts/* 4 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/SpillAndLoadManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap; 20 | 21 | /** 22 | * Interface for spill and load manager. 23 | */ 24 | public interface SpillAndLoadManager { 25 | 26 | /** 27 | * Check resource to decide whether to spill or load state. 28 | */ 29 | void checkResource(); 30 | } 31 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/space/Constants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap.space; 20 | 21 | /** 22 | * constants. 23 | */ 24 | public class Constants { 25 | public static final int NO_SPACE = -1; 26 | public static final int BUCKET_SIZE = 1024 * 1024; 27 | public static final int FOUR_BYTES_BITS = 32; 28 | public static final long FOUR_BYTES_MARK = 0xFFFFFFFFL; 29 | } 30 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .cache 2 | scalastyle-output.xml 3 | .classpath 4 | .idea 5 | .metadata 6 | .settings 7 | .project 8 | .version.properties 9 | filter.properties 10 | logs.zip 11 | target 12 | tmp 13 | *.class 14 | *.iml 15 | *.swp 16 | *.jar 17 | *.zip 18 | *.log 19 | *.pyc 20 | .DS_Store 21 | build-target 22 | flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/avro/ 23 | flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/generated/ 24 | flink-formats/flink-parquet/src/test/java/org/apache/flink/formats/parquet/generated/ 25 | flink-runtime-web/web-dashboard/node/ 26 | flink-runtime-web/web-dashboard/node_modules/ 27 | flink-runtime-web/web-dashboard/web/ 28 | flink-python/dist/ 29 | flink-python/build/ 30 | flink-python/pyflink.egg-info/ 31 | flink-python/apache_flink.egg-info/ 32 | flink-python/docs/_build 33 | flink-python/.tox/ 34 | flink-python/dev/download 35 | flink-python/dev/.conda/ 36 | flink-python/dev/log/ 37 | flink-python/dev/.stage.txt 38 | flink-python/.eggs/ 39 | flink-python/**/*.c 40 | flink-python/**/*.so 41 | atlassian-ide-plugin.xml 42 | out/ 43 | /docs/api 44 | /docs/content 45 | /docs/.bundle 46 | /docs/.rubydeps 47 | /docs/ruby2/.bundle 48 | /docs/ruby2/.rubydeps 49 | /docs/.jekyll-metadata 50 | *.ipr 51 | *.iws 52 | tools/flink 53 | tools/flink-* 54 | tools/releasing/release 55 | tools/japicmp-output 56 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/space/AllocateStrategy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap.space; 20 | 21 | /** 22 | * The strategy for space allocation. 23 | */ 24 | public enum AllocateStrategy { 25 | 26 | /** 27 | * This strategy implements a simple buddy-like allocator used for small space. 28 | */ 29 | SmallBucket, 30 | 31 | /** 32 | * This strategy is used to allocate large space, and reduce fragments. 33 | */ 34 | HugeBucket 35 | } 36 | -------------------------------------------------------------------------------- /flink-spillable-benchmark/README.md: -------------------------------------------------------------------------------- 1 | # Spillable Benchmark 2 | 3 | This benchmark uses a simple word count to compare performance of different state backends. 4 | 5 | ## Building 6 | 7 | ``` 8 | git clone https://github.com/realtime-storage-engine/flink-spillable-statebackend.git 9 | cd flink-spillable-statebackend/flink-spillable-benchmark 10 | mvn clean package 11 | ``` 12 | 13 | Benchmark JAR file (`flink-spillable-benchmark-1.0-SNAPSHOT.jar`) will be installed into `target` directory. 14 | 15 | ## Options 16 | 17 | * jobName - name of the job 18 | * wordNumber - number of different keys 19 | * wordLength - length of `String` word 20 | * wordRate - rate of source to emit words 21 | * checkpointInterval - the checkpoint interval, in milliseconds 22 | 23 | ## Run Benchmark 24 | 25 | 1. set `state.backend` and backend-related configurations in flink-conf.yaml 26 | 27 | 2. submit job to yarn cluster on per-job mode as following 28 | 29 | ```bash 30 | export jobName="spillable-bench" 31 | export wordNumber="20000000" 32 | export wordLength="16" 33 | export wordRate="1000000" 34 | export checkpointInterval="60000" 35 | export mainClass="org.apache.flink.spillable.benchmark.WordCount" 36 | 37 | ./bin/flink run -d -m yarn-cluster -ynm ${jobName} -c ${mainClass} /path/to/benchmark-jar -jobName ${jobName} -checkpointInterval ${checkpointInterval} -wordNumber ${wordNumber} -wordRate ${wordRate} -wordLength ${wordLength} 38 | ``` -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/space/ChunkAllocator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap.space; 20 | 21 | import java.io.Closeable; 22 | 23 | /** 24 | * Allocator to create chunks. 25 | */ 26 | public interface ChunkAllocator extends Closeable { 27 | 28 | /** 29 | * Creates a chunk with the id and strategy. 30 | * 31 | * @param chunkId id of the chunk. 32 | * @param allocateStrategy strategy to allocate chunk. 33 | * @return a chunk. 34 | */ 35 | Chunk createChunk(int chunkId, AllocateStrategy allocateStrategy); 36 | } 37 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/test/resources/log4j2-test.properties: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | ################################################################################ 18 | 19 | # Set root logger level to OFF to not flood build logs 20 | # set manually to INFO for debugging purposes 21 | rootLogger.level = OFF 22 | rootLogger.appenderRef.test.ref = TestLogger 23 | 24 | appender.testlogger.name = TestLogger 25 | appender.testlogger.type = CONSOLE 26 | appender.testlogger.target = SYSTEM_ERR 27 | appender.testlogger.layout.type = PatternLayout 28 | appender.testlogger.layout.pattern = %-4r [%t] %-5p %c %x - %m%n 29 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/space/DirectBufferChunkAllocator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap.space; 20 | 21 | import org.apache.flink.core.memory.MemorySegment; 22 | import org.apache.flink.core.memory.MemorySegmentFactory; 23 | 24 | /** 25 | * Manages chunks allocated from direct buffer. 26 | */ 27 | class DirectBufferChunkAllocator extends AbstractChunkAllocator { 28 | 29 | DirectBufferChunkAllocator(int chunkSize) { 30 | super(chunkSize); 31 | } 32 | 33 | @Override 34 | MemorySegment allocate(int chunkSize) { 35 | return MemorySegmentFactory.allocateUnpooledOffHeapMemory(chunkSize); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/space/HeapBufferChunkAllocator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.flink.runtime.state.heap.space; 19 | 20 | import org.apache.flink.core.memory.MemorySegment; 21 | import org.apache.flink.core.memory.MemorySegmentFactory; 22 | 23 | /** 24 | * Manages chunks allocated from heap. This is mainly used for test. 25 | */ 26 | class HeapBufferChunkAllocator extends AbstractChunkAllocator { 27 | 28 | HeapBufferChunkAllocator(int chunkSize) { 29 | super(chunkSize); 30 | } 31 | 32 | @Override 33 | MemorySegment allocate(int chunkSize) { 34 | return MemorySegmentFactory.allocateUnpooledSegment(chunkSize); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/space/SpaceConstants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap.space; 20 | 21 | /** 22 | * Some constants used for space. 23 | */ 24 | public class SpaceConstants { 25 | 26 | private SpaceConstants() { 27 | } 28 | 29 | /** This indicates there is no space left. */ 30 | static final int NO_SPACE = -1; 31 | 32 | /** Size of bucket. */ 33 | static final int BUCKET_SIZE = 1024 * 1024; 34 | 35 | /** Number of bits for four bytes. */ 36 | public static final int FOUR_BYTES_BITS = 32; 37 | 38 | /** Mask for four bytes. */ 39 | public static final long FOUR_BYTES_MARK = 0xFFFFFFFFL; 40 | } 41 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/NodeStatus.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap; 20 | 21 | /** 22 | * Status of the node. 23 | */ 24 | public enum NodeStatus { 25 | 26 | PUT((byte) 0), REMOVE((byte) 1); 27 | 28 | private final byte value; 29 | 30 | NodeStatus(byte value) { 31 | this.value = value; 32 | } 33 | 34 | public byte getValue() { 35 | return value; 36 | } 37 | 38 | public static NodeStatus valueOf(byte value) { 39 | switch (value) { 40 | case 0: 41 | return PUT; 42 | case 1: 43 | return REMOVE; 44 | default: 45 | throw new IllegalArgumentException("Unknown type: " + value); 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/space/BucketAllocator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap.space; 20 | 21 | /** 22 | * Bucket is a logic space in the chunk. Each Chunk has a 23 | * {@link BucketAllocator} to avoid space fragments. 24 | */ 25 | public interface BucketAllocator { 26 | 27 | /** 28 | * Allocate a space in a the chunk. 29 | * 30 | * @param size size of space to allocate. 31 | * @return offset of allocated space in the chunk. 32 | */ 33 | int allocate(int size); 34 | 35 | /** 36 | * Free a space with the given offset in the chunk. 37 | * 38 | * @param offset offset of space in the chunk. 39 | */ 40 | void free(int offset); 41 | } 42 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/estimate/StateMemoryEstimator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap.estimate; 20 | 21 | /** 22 | * Interface to estimate the memory size of state. 23 | * 24 | * @param type of key 25 | * @param type of namespace 26 | * @param type of state 27 | */ 28 | public interface StateMemoryEstimator { 29 | 30 | /** 31 | * Update estimated memory size. 32 | * 33 | * @param key the key to estimate 34 | * @param namespace the namespace to estimate 35 | * @param state the state to estimate 36 | */ 37 | void updateEstimatedSize(K key, N namespace, S state); 38 | 39 | /** 40 | * Return the estimated size. 41 | * 42 | * @return estimated size. -1 indicates an invalid estimation 43 | */ 44 | long getEstimatedSize(); 45 | } 46 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/test/java/org/apache/flink/runtime/state/ttl/SpillableTtlUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.ttl; 20 | 21 | import java.util.Arrays; 22 | import java.util.List; 23 | 24 | /** 25 | * Utilities for ttl state tests. 26 | */ 27 | public class SpillableTtlUtils { 28 | 29 | public static List> getContexts() { 30 | return Arrays.asList( 31 | new TtlValueStateTestContext(), 32 | new TtlFixedLenElemListStateTestContext(), 33 | new TtlNonFixedLenElemListStateTestContext(), 34 | new SpillableTtlMapStateAllEntriesTestContext(), 35 | new TtlMapStatePerElementTestContext(), 36 | new TtlMapStatePerNullElementTestContext(), 37 | new TtlAggregatingStateTestContext(), 38 | new TtlReducingStateTestContext(), 39 | new TtlFoldingStateTestContext()); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/test/java/org/apache/flink/runtime/state/heap/TestHeapStatusMonitor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap; 20 | 21 | /** 22 | * Heap status monitor used for tests. 23 | */ 24 | public class TestHeapStatusMonitor extends HeapStatusMonitor { 25 | 26 | private final long maxMemory; 27 | 28 | private MonitorResult monitorResult; 29 | 30 | public TestHeapStatusMonitor(long maxMemory) { 31 | // disable periodically check 32 | super(Long.MAX_VALUE); 33 | this.maxMemory = maxMemory; 34 | } 35 | 36 | public void setMonitorResult(MonitorResult monitorResult) { 37 | this.monitorResult = monitorResult; 38 | } 39 | 40 | @Override 41 | public MonitorResult getMonitorResult() { 42 | return monitorResult; 43 | } 44 | 45 | @Override 46 | public long getMaxMemory() { 47 | return maxMemory; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/space/AbstractChunk.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap.space; 20 | 21 | import org.apache.flink.util.Preconditions; 22 | 23 | /** 24 | * Base implementation of {@link Chunk}. 25 | */ 26 | public abstract class AbstractChunk implements Chunk { 27 | 28 | /** Id of the chunk. */ 29 | private final int chunkId; 30 | 31 | /** Capacity of the chunk. */ 32 | final int capacity; 33 | 34 | AbstractChunk(int chunkId, int capacity) { 35 | this.chunkId = chunkId; 36 | Preconditions.checkArgument((capacity & capacity - 1) == 0, 37 | "Capacity of chunk should be a power of 2, but the actual is " + capacity); 38 | this.capacity = capacity; 39 | } 40 | 41 | @Override 42 | public int getChunkId() { 43 | return chunkId; 44 | } 45 | 46 | @Override 47 | public int getChunkCapacity() { 48 | return capacity; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/space/Allocator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap.space; 20 | 21 | import java.io.Closeable; 22 | 23 | /** 24 | * Implementations are responsible for allocate space. 25 | */ 26 | public interface Allocator extends Closeable { 27 | 28 | /** 29 | * Allocate space with the given size. 30 | * 31 | * @param size size of space to allocate. 32 | * @return address of the allocated space. 33 | * @throws Exception This method will throw exception if failed to allocate space. 34 | */ 35 | long allocate(int size) throws Exception; 36 | 37 | /** 38 | * Free the space with the given address. 39 | * 40 | * @param address address of the space to free. 41 | */ 42 | void free(long address); 43 | 44 | /** 45 | * Returns the chunk with the given chunk id. 46 | * 47 | * @param chunkId id of the chunk. 48 | * @return chunk with the given id. 49 | */ 50 | Chunk getChunkById(int chunkId); 51 | } 52 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/test/java/org/apache/flink/runtime/state/ttl/SpillableTtlMapStateAllEntriesTestContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.ttl; 20 | 21 | import java.util.Collections; 22 | import java.util.HashMap; 23 | import java.util.Map; 24 | import java.util.Set; 25 | 26 | /** Test suite for collection methods of {@link TtlMapState}. */ 27 | public class SpillableTtlMapStateAllEntriesTestContext extends TtlMapStateAllEntriesTestContext { 28 | 29 | @Override 30 | public Object getOriginal() throws Exception { 31 | Iterable>> iterable = ttlState.original.entries(); 32 | if (iterable == null) { 33 | return Collections.emptySet(); 34 | } 35 | 36 | if (iterable instanceof Set) { 37 | return iterable; 38 | } 39 | 40 | Map> map = new HashMap<>(); 41 | for (Map.Entry> entry : iterable) { 42 | map.put(entry.getKey(), entry.getValue()); 43 | } 44 | return map.entrySet(); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/test/java/org/apache/flink/runtime/state/heap/SpillableStateBackendMigrationTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * * Licensed to the Apache Software Foundation (ASF) under one 4 | * * or more contributor license agreements. See the NOTICE file 5 | * * distributed with this work for additional information 6 | * * regarding copyright ownership. The ASF licenses this file 7 | * * to you under the Apache License, Version 2.0 (the 8 | * * "License"); you may not use this file except in compliance 9 | * * with the License. You may obtain a copy of the License at 10 | * * 11 | * * http://www.apache.org/licenses/LICENSE-2.0 12 | * * 13 | * * Unless required by applicable law or agreed to in writing, software 14 | * * distributed under the License is distributed on an "AS IS" BASIS, 15 | * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | * * See the License for the specific language governing permissions and 17 | * * limitations under the License. 18 | * 19 | */ 20 | 21 | package org.apache.flink.runtime.state.heap; 22 | 23 | import org.apache.flink.runtime.state.StateBackendMigrationTestBase; 24 | import org.apache.flink.runtime.state.filesystem.FsStateBackend; 25 | 26 | import java.io.File; 27 | 28 | /** 29 | * Tests for the partitioned state part of {@link SpillableStateBackend}. 30 | */ 31 | public class SpillableStateBackendMigrationTest extends StateBackendMigrationTestBase { 32 | 33 | @Override 34 | protected SpillableStateBackend getStateBackend() throws Exception { 35 | File checkpointPath = tempFolder.newFolder(); 36 | SpillableStateBackend backend = new SpillableStateBackend(new FsStateBackend(checkpointPath.toURI(), true)); 37 | 38 | String dbPath = tempFolder.newFolder().getAbsolutePath(); 39 | backend.setDbStoragePaths(dbPath); 40 | 41 | return backend; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/space/SpaceUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap.space; 20 | 21 | import static org.apache.flink.runtime.state.heap.space.SpaceConstants.FOUR_BYTES_BITS; 22 | import static org.apache.flink.runtime.state.heap.space.SpaceConstants.FOUR_BYTES_MARK; 23 | 24 | /** 25 | * Utilities for space. 26 | */ 27 | public class SpaceUtils { 28 | 29 | /** 30 | * Returns the id of chunk used by the space with the given space. 31 | * 32 | * @param address address of the space. 33 | * @return id of chunk used by space. 34 | */ 35 | public static int getChunkIdByAddress(long address) { 36 | return (int) ((address >>> FOUR_BYTES_BITS) & FOUR_BYTES_MARK); 37 | } 38 | 39 | /** 40 | * Returns the offset of space in the chunk. 41 | * 42 | * @param address address of the space. 43 | * @return id of chunk used by space. 44 | */ 45 | public static int getChunkOffsetByAddress(long address) { 46 | return (int) (address & FOUR_BYTES_MARK); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/LevelIndexHeader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap; 20 | 21 | /** 22 | * Head level index for skip list. 23 | */ 24 | public interface LevelIndexHeader { 25 | 26 | /** 27 | * Returns the top level of skip list. 28 | * 29 | * @return the top level of skip list. 30 | */ 31 | int getLevel(); 32 | 33 | /** 34 | * Updates the top level of skip list to the given level. 35 | * 36 | * @param level the level which top level of skip list updates to. 37 | */ 38 | void updateLevel(int level); 39 | 40 | /** 41 | * Returns the next node in the given level. 42 | * 43 | * @param level the level whose next node is returned. 44 | * @return id of the next node. 45 | */ 46 | long getNextNode(int level); 47 | 48 | /** 49 | * Updates the next node in the given level to the specified node id. 50 | * 51 | * @param level the level whose next node is updated. 52 | * @param newNodeId the id of the next node. 53 | */ 54 | void updateNextNode(int level, long newNodeId); 55 | } 56 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/SpillableStateBackendFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * * Licensed to the Apache Software Foundation (ASF) under one 4 | * * or more contributor license agreements. See the NOTICE file 5 | * * distributed with this work for additional information 6 | * * regarding copyright ownership. The ASF licenses this file 7 | * * to you under the Apache License, Version 2.0 (the 8 | * * "License"); you may not use this file except in compliance 9 | * * with the License. You may obtain a copy of the License at 10 | * * 11 | * * http://www.apache.org/licenses/LICENSE-2.0 12 | * * 13 | * * Unless required by applicable law or agreed to in writing, software 14 | * * distributed under the License is distributed on an "AS IS" BASIS, 15 | * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | * * See the License for the specific language governing permissions and 17 | * * limitations under the License. 18 | * 19 | */ 20 | 21 | package org.apache.flink.runtime.state.heap; 22 | 23 | import org.apache.flink.configuration.CheckpointingOptions; 24 | import org.apache.flink.configuration.IllegalConfigurationException; 25 | import org.apache.flink.configuration.ReadableConfig; 26 | import org.apache.flink.runtime.state.StateBackendFactory; 27 | 28 | import java.io.IOException; 29 | 30 | /** 31 | * A factory that creates an {@link SpillableStateBackend} from a configuration. 32 | */ 33 | public class SpillableStateBackendFactory implements StateBackendFactory { 34 | 35 | @Override 36 | public SpillableStateBackend createFromConfig(ReadableConfig config, ClassLoader classLoader) 37 | throws IllegalConfigurationException, IOException { 38 | final String checkpointDirURI = config.get(CheckpointingOptions.CHECKPOINTS_DIRECTORY); 39 | return new SpillableStateBackend(checkpointDirURI).configure(config, classLoader); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/estimate/ValueStateMemoryEstimator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap.estimate; 20 | 21 | import org.apache.flink.api.common.state.ValueState; 22 | import org.apache.flink.api.common.typeutils.TypeSerializer; 23 | 24 | /** 25 | * Estimates memory usage of {@link ValueState}. 26 | * 27 | * @param type of key 28 | * @param type of namespace 29 | * @param type of state 30 | */ 31 | public class ValueStateMemoryEstimator extends AbstractStateMemoryEstimator { 32 | 33 | private boolean isStateFix; 34 | private long stateSize; 35 | 36 | public ValueStateMemoryEstimator( 37 | TypeSerializer keySerializer, 38 | TypeSerializer namespaceSerializer, 39 | TypeSerializer valueSerializer) { 40 | super(keySerializer, namespaceSerializer); 41 | this.isStateFix = valueSerializer.getLength() != -1; 42 | this.stateSize = -1; 43 | } 44 | 45 | @Override 46 | protected long getStateSize(V state) { 47 | if (stateSize == -1 || !isStateFix) { 48 | this.stateSize = RamUsageEstimator.sizeOf(state); 49 | } 50 | 51 | return this.stateSize; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /flink-spillable-benchmark/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 21 | 24 | 25 | 4.0.0 26 | 27 | 28 | org.apache.flink 29 | flink-spillable-statebackend 30 | 1.0-SNAPSHOT 31 | 32 | 33 | flink-spillable-benchmark 34 | flink-spillable-benchmark 35 | jar 36 | 37 | 38 | 39 | org.apache.flink 40 | flink-streaming-java_${scala.binary.version} 41 | ${flink.version} 42 | provided 43 | 44 | 45 | 46 | org.apache.flink 47 | flink-core 48 | ${flink.version} 49 | provided 50 | 51 | 52 | 53 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/CheckpointManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap; 20 | 21 | import org.apache.flink.runtime.state.KeyedStateHandle; 22 | import org.apache.flink.runtime.state.SnapshotResult; 23 | 24 | import java.util.concurrent.RunnableFuture; 25 | 26 | /** 27 | * Manages running checkpoints. 28 | */ 29 | public interface CheckpointManager { 30 | 31 | /** 32 | * Register the checkpoint to run. 33 | * 34 | * @param checkpointId id of checkpoint to register 35 | * @param runnableFuture future to run 36 | * 37 | * @return true if register successfully 38 | */ 39 | boolean registerCheckpoint(long checkpointId, RunnableFuture> runnableFuture); 40 | 41 | /** 42 | * Unregister a checkpoint. 43 | * 44 | * @param checkpointId id of checkpoint to unregister 45 | * @return true if unregister successfully 46 | */ 47 | boolean unRegisterCheckpoint(long checkpointId); 48 | 49 | /** 50 | * Cancel a checkpoint. 51 | * 52 | * @param checkpointId id of checkpoint to cancel 53 | * @return true if cancel successfully 54 | */ 55 | boolean cancelCheckpoint(long checkpointId); 56 | 57 | /** 58 | * Cancel all running checkpoints. 59 | */ 60 | void cancelAllCheckpoints(); 61 | } 62 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/SpillableSnapshotStrategySynchronicityBehavior.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * * Licensed to the Apache Software Foundation (ASF) under one 4 | * * or more contributor license agreements. See the NOTICE file 5 | * * distributed with this work for additional information 6 | * * regarding copyright ownership. The ASF licenses this file 7 | * * to you under the Apache License, Version 2.0 (the 8 | * * "License"); you may not use this file except in compliance 9 | * * with the License. You may obtain a copy of the License at 10 | * * 11 | * * http://www.apache.org/licenses/LICENSE-2.0 12 | * * 13 | * * Unless required by applicable law or agreed to in writing, software 14 | * * distributed under the License is distributed on an "AS IS" BASIS, 15 | * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | * * See the License for the specific language governing permissions and 17 | * * limitations under the License. 18 | * 19 | */ 20 | 21 | package org.apache.flink.runtime.state.heap; 22 | 23 | import org.apache.flink.api.common.typeutils.TypeSerializer; 24 | import org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo; 25 | import org.apache.flink.runtime.state.heap.space.SpaceAllocator; 26 | 27 | class SpillableSnapshotStrategySynchronicityBehavior implements SnapshotStrategySynchronicityBehavior { 28 | 29 | private final SpaceAllocator spaceAllocator; 30 | private final SpillAndLoadManager spillAndLoadManager; 31 | 32 | SpillableSnapshotStrategySynchronicityBehavior( 33 | SpaceAllocator spaceAllocator, 34 | SpillAndLoadManager spillAndLoadManager) { 35 | this.spaceAllocator = spaceAllocator; 36 | this.spillAndLoadManager = spillAndLoadManager; 37 | } 38 | 39 | @Override 40 | public boolean isAsynchronous() { 41 | return true; 42 | } 43 | 44 | @Override 45 | public StateTable newStateTable( 46 | InternalKeyContext keyContext, 47 | RegisteredKeyValueStateBackendMetaInfo newMetaInfo, 48 | TypeSerializer keySerializer) { 49 | return new SpillableStateTableImpl<>(keyContext, newMetaInfo, keySerializer, 50 | spaceAllocator, spillAndLoadManager); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 21 | 24 | 25 | 4.0.0 26 | 27 | org.apache.flink 28 | flink-spillable-statebackend 29 | 1.0-SNAPSHOT 30 | 31 | flink-spillable-statebackend 32 | pom 33 | 34 | 35 | 36 | The Apache Software License, Version 2.0 37 | https://www.apache.org/licenses/LICENSE-2.0.txt 38 | repo 39 | 40 | 41 | 42 | 43 | UTF-8 44 | UTF-8 45 | 1.8 46 | ${java.version} 47 | ${java.version} 48 | 1.11-SNAPSHOT 49 | 2.11 50 | 51 | 52 | 53 | flink-statebackend-heap-spillable 54 | flink-spillable-benchmark 55 | 56 | 57 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/estimate/AbstractStateMemoryEstimator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap.estimate; 20 | 21 | import org.apache.flink.api.common.typeutils.TypeSerializer; 22 | 23 | /** 24 | * Base class for state memory estimation. 25 | * 26 | * @param type of key 27 | * @param type of namespace 28 | * @param type of state 29 | */ 30 | public abstract class AbstractStateMemoryEstimator implements StateMemoryEstimator { 31 | 32 | private boolean isKeyFix; 33 | private long keySize; 34 | 35 | private boolean isNamespaceFix; 36 | private long namespaceSize; 37 | 38 | private long stateSize; 39 | 40 | public AbstractStateMemoryEstimator( 41 | TypeSerializer keySerializer, 42 | TypeSerializer namespaceSerializer) { 43 | this.isKeyFix = keySerializer.getLength() != -1; 44 | this.keySize = -1; 45 | this.isNamespaceFix = namespaceSerializer.getLength() != -1; 46 | this.namespaceSize = -1; 47 | } 48 | 49 | @Override 50 | public void updateEstimatedSize(K key, N namespace, S state) { 51 | if (keySize == -1 || !isKeyFix) { 52 | this.keySize = RamUsageEstimator.sizeOf(key); 53 | } 54 | 55 | if (namespaceSize == -1 || !isNamespaceFix) { 56 | this.namespaceSize = RamUsageEstimator.sizeOf(key); 57 | } 58 | 59 | this.stateSize = getStateSize(state); 60 | } 61 | 62 | @Override 63 | public long getEstimatedSize() { 64 | return keySize + namespaceSize + stateSize; 65 | } 66 | 67 | protected abstract long getStateSize(S state); 68 | } 69 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/estimate/SampleStateMemoryEstimator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap.estimate; 20 | 21 | import org.apache.flink.util.Preconditions; 22 | 23 | /** 24 | * Sample states to estimate memory usage. 25 | */ 26 | public class SampleStateMemoryEstimator implements StateMemoryEstimator { 27 | 28 | private final StateMemoryEstimator stateMemoryEstimator; 29 | private final int sampleCount; 30 | 31 | private long count; 32 | private long totalSize; 33 | private long totalNum; 34 | 35 | public SampleStateMemoryEstimator( 36 | StateMemoryEstimator stateMemoryEstimator, 37 | int sampleCount) { 38 | this.stateMemoryEstimator = Preconditions.checkNotNull(stateMemoryEstimator); 39 | Preconditions.checkArgument(sampleCount > 0, "Sample count should be positive"); 40 | this.sampleCount = sampleCount; 41 | this.count = 0; 42 | this.totalSize = 0; 43 | this.totalNum = 0; 44 | } 45 | 46 | @Override 47 | public void updateEstimatedSize(K key, N namespace, S state) { 48 | if (count++ % sampleCount == 0) { 49 | forceUpdateEstimatedSize(key, namespace, state); 50 | } 51 | } 52 | 53 | @Override 54 | public long getEstimatedSize() { 55 | return totalNum > 0 ? totalSize / totalNum : -1; 56 | } 57 | 58 | public void forceUpdateEstimatedSize(K key, N namespace, S state) { 59 | stateMemoryEstimator.updateEstimatedSize(key, namespace, state); 60 | totalSize += stateMemoryEstimator.getEstimatedSize(); 61 | totalNum++; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/space/DefaultChunkImpl.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap.space; 20 | 21 | import org.apache.flink.core.memory.MemorySegment; 22 | 23 | import static org.apache.flink.runtime.state.heap.space.SpaceConstants.BUCKET_SIZE; 24 | 25 | /** 26 | * Default implementation of {@link Chunk}. 27 | */ 28 | public class DefaultChunkImpl extends AbstractChunk { 29 | 30 | /** 31 | * The backed memory segment. 32 | */ 33 | private final MemorySegment memorySegment; 34 | 35 | /** 36 | * Bucket allocator used for this chunk. 37 | */ 38 | private final BucketAllocator bucketAllocator; 39 | 40 | public DefaultChunkImpl(int chunkId, MemorySegment memorySegment, AllocateStrategy allocateStrategy) { 41 | super(chunkId, memorySegment.size()); 42 | this.memorySegment = memorySegment; 43 | switch (allocateStrategy) { 44 | case SmallBucket: 45 | this.bucketAllocator = new PowerTwoBucketAllocator(capacity, BUCKET_SIZE); 46 | break; 47 | default: 48 | this.bucketAllocator = new DirectBucketAllocator(capacity); 49 | } 50 | } 51 | 52 | @Override 53 | public int allocate(int len) { 54 | return bucketAllocator.allocate(len); 55 | } 56 | 57 | @Override 58 | public void free(int interChunkOffset) { 59 | bucketAllocator.free(interChunkOffset); 60 | } 61 | 62 | @Override 63 | public int getOffsetInSegment(int offsetInChunk) { 64 | return offsetInChunk; 65 | } 66 | 67 | @Override 68 | public MemorySegment getMemorySegment(int chunkOffset) { 69 | return memorySegment; 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/test/java/org/apache/flink/runtime/state/heap/ttl/SpillableSnapshotTtlStateTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * * Licensed to the Apache Software Foundation (ASF) under one 4 | * * or more contributor license agreements. See the NOTICE file 5 | * * distributed with this work for additional information 6 | * * regarding copyright ownership. The ASF licenses this file 7 | * * to you under the Apache License, Version 2.0 (the 8 | * * "License"); you may not use this file except in compliance 9 | * * with the License. You may obtain a copy of the License at 10 | * * 11 | * * http://www.apache.org/licenses/LICENSE-2.0 12 | * * 13 | * * Unless required by applicable law or agreed to in writing, software 14 | * * distributed under the License is distributed on an "AS IS" BASIS, 15 | * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | * * See the License for the specific language governing permissions and 17 | * * limitations under the License. 18 | * 19 | */ 20 | 21 | package org.apache.flink.runtime.state.heap.ttl; 22 | 23 | import org.apache.flink.runtime.state.StateBackend; 24 | import org.apache.flink.runtime.state.heap.SpillableStateBackend; 25 | import org.apache.flink.runtime.state.memory.MemoryStateBackend; 26 | import org.apache.flink.runtime.state.ttl.SpillableTtlUtils; 27 | import org.apache.flink.runtime.state.ttl.StateBackendTestContext; 28 | import org.apache.flink.runtime.state.ttl.TtlStateTestBase; 29 | import org.apache.flink.runtime.state.ttl.TtlStateTestContextBase; 30 | import org.apache.flink.runtime.state.ttl.TtlTimeProvider; 31 | 32 | import org.junit.runners.Parameterized; 33 | 34 | import java.util.List; 35 | 36 | /** 37 | * Test suite for spillable state TTL. 38 | */ 39 | public class SpillableSnapshotTtlStateTest extends TtlStateTestBase { 40 | 41 | @Override 42 | protected StateBackendTestContext createStateBackendTestContext(TtlTimeProvider timeProvider) { 43 | return new StateBackendTestContext(timeProvider) { 44 | @Override 45 | protected StateBackend createStateBackend() { 46 | return new SpillableStateBackend(new MemoryStateBackend(true)); 47 | } 48 | }; 49 | } 50 | 51 | @Override 52 | protected boolean incrementalCleanupSupported() { 53 | return true; 54 | } 55 | 56 | @Parameterized.Parameters(name = "{0}") 57 | public static List> testContexts() { 58 | return SpillableTtlUtils.getContexts(); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/space/Chunk.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap.space; 20 | 21 | import org.apache.flink.core.memory.MemorySegment; 22 | 23 | /** 24 | * Chunk is a logically contiguous space backed by one or multiple {@link MemorySegment}. 25 | *

26 | * The backing MemorySegment may wrap an on-heap byte array, an off-heap {@link java.nio.DirectByteBuffer}, 27 | * or a {@link java.nio.MappedByteBuffer} from a memory-mapped file. 28 | */ 29 | public interface Chunk { 30 | /** 31 | * Try to allocate size bytes from the chunk. 32 | * 33 | * @param len size of bytes to allocate. 34 | * @return the offset of the successful allocation, or -1 to indicate not-enough-space 35 | */ 36 | int allocate(int len); 37 | 38 | /** 39 | * release the space addressed by interChunkOffset. 40 | * 41 | * @param interChunkOffset offset of the chunk 42 | */ 43 | void free(int interChunkOffset); 44 | 45 | /** 46 | * @return Id of this Chunk 47 | */ 48 | int getChunkId(); 49 | 50 | int getChunkCapacity(); 51 | 52 | /** 53 | * Returns the backed {@link MemorySegment} for the space with the offset. 54 | * 55 | * @param offsetInChunk offset of space in the chunk. 56 | * @return memory segment backed the space. 57 | */ 58 | MemorySegment getMemorySegment(int offsetInChunk); 59 | 60 | /** 61 | * Returns the offset of the space in the backed {@link MemorySegment}. 62 | * 63 | * @param offsetInChunk offset of space in the chunk. 64 | * @return offset of space in the memory segment. 65 | */ 66 | int getOffsetInSegment(int offsetInChunk); 67 | } 68 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/space/AbstractChunkAllocator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap.space; 20 | 21 | import org.apache.flink.core.memory.MemorySegment; 22 | 23 | import org.slf4j.Logger; 24 | import org.slf4j.LoggerFactory; 25 | 26 | import java.util.ArrayList; 27 | import java.util.concurrent.atomic.AtomicBoolean; 28 | 29 | /** 30 | * An abstract base implementation of the {@link ChunkAllocator} interface. 31 | */ 32 | public abstract class AbstractChunkAllocator implements ChunkAllocator { 33 | private static final Logger LOG = LoggerFactory.getLogger(AbstractChunkAllocator.class); 34 | 35 | private final int chunkSize; 36 | 37 | private ArrayList segments; 38 | 39 | private AtomicBoolean closed; 40 | 41 | AbstractChunkAllocator(int chunkSize) { 42 | this.chunkSize = chunkSize; 43 | this.segments = new ArrayList<>(); 44 | this.closed = new AtomicBoolean(false); 45 | } 46 | 47 | @Override 48 | public Chunk createChunk(int chunkId, AllocateStrategy allocateStrategy) { 49 | MemorySegment segment = allocate(chunkSize); 50 | segments.add(segment); 51 | return new DefaultChunkImpl(chunkId, segment, allocateStrategy); 52 | } 53 | 54 | @Override 55 | public void close() { 56 | if (closed.compareAndSet(false, true)) { 57 | segments.forEach(MemorySegment::free); 58 | segments.clear(); 59 | } else { 60 | LOG.warn("This chunk allocator {} has been already closed.", this); 61 | } 62 | } 63 | 64 | /** 65 | * Allocate a buffer for the chunk. 66 | * 67 | * @param chunkSize the size of the chunk to allocate. 68 | * @return the buffer for the chunk. 69 | */ 70 | abstract MemorySegment allocate(int chunkSize); 71 | } 72 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/test/java/org/apache/flink/runtime/state/heap/SpillableOptionsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap; 20 | 21 | import org.apache.flink.configuration.ConfigOption; 22 | import org.apache.flink.configuration.Configuration; 23 | 24 | import org.junit.Test; 25 | import org.powermock.core.IdentityHashSet; 26 | 27 | import java.lang.reflect.Field; 28 | import java.lang.reflect.Modifier; 29 | import java.util.Set; 30 | 31 | import static org.junit.Assert.assertEquals; 32 | import static org.junit.Assert.assertTrue; 33 | 34 | /** 35 | * Tests for {@link SpillableOptions}. 36 | */ 37 | public class SpillableOptionsTest { 38 | 39 | @Test 40 | public void testSupportedConfig() throws Exception { 41 | IdentityHashSet optionSet = getOptionsViaReflection(); 42 | 43 | assertEquals(optionSet.size(), SpillableOptions.SUPPORTED_CONFIG.length); 44 | for (ConfigOption option : SpillableOptions.SUPPORTED_CONFIG) { 45 | assertTrue(optionSet.contains(option)); 46 | } 47 | } 48 | 49 | @Test 50 | public void testFilterOptions() throws Exception { 51 | IdentityHashSet optionSet = getOptionsViaReflection(); 52 | 53 | Set keySet = SpillableOptions.filter(new Configuration()).toMap().keySet(); 54 | assertEquals(optionSet.size(), keySet.size()); 55 | for (ConfigOption option : optionSet) { 56 | assertTrue(keySet.contains(option.key())); 57 | } 58 | } 59 | 60 | private IdentityHashSet getOptionsViaReflection() throws Exception { 61 | IdentityHashSet set = new IdentityHashSet<>(); 62 | Field[] fields = SpillableOptions.class.getDeclaredFields(); 63 | for (Field f : fields) { 64 | if (Modifier.isStatic(f.getModifiers()) && f.getType() == ConfigOption.class) { 65 | set.add((ConfigOption) f.get(null)); 66 | } 67 | } 68 | 69 | return set; 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/test/java/org/apache/flink/runtime/state/heap/space/MmapChunkAllocatorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, 15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | * See the License for the specific language governing permissions and 17 | * limitations under the License. 18 | */ 19 | 20 | package org.apache.flink.runtime.state.heap.space; 21 | 22 | import org.apache.flink.configuration.Configuration; 23 | import org.apache.flink.util.FlinkRuntimeException; 24 | 25 | import org.junit.Assert; 26 | import org.junit.Rule; 27 | import org.junit.Test; 28 | import org.junit.rules.TemporaryFolder; 29 | 30 | import java.io.File; 31 | 32 | /** 33 | * Tests for {@link MmapChunkAllocator}. 34 | */ 35 | public class MmapChunkAllocatorTest { 36 | 37 | @Rule 38 | public final TemporaryFolder tempFolder = new TemporaryFolder(); 39 | 40 | @Test 41 | public void testPrepareDirectory() throws Exception { 42 | File[] dirs = new File[3]; 43 | for (int i = 0; i < dirs.length; i++) { 44 | dirs[i] = tempFolder.newFolder(); 45 | } 46 | 47 | MmapChunkAllocator allocator = 48 | new MmapChunkAllocator(10240, dirs, new Configuration()); 49 | for (File dir : dirs) { 50 | Assert.assertTrue(new File(dir, MmapChunkAllocator.MMAP_DIR).exists()); 51 | } 52 | 53 | allocator.close(); 54 | } 55 | 56 | @Test 57 | public void testPrepareDirectoryFailed() throws Exception { 58 | File[] dirs = new File[3]; 59 | dirs[0] = tempFolder.newFolder(); 60 | dirs[1] = tempFolder.newFolder(); 61 | // deliver a file to make preparation failed 62 | dirs[2] = tempFolder.newFile(); 63 | 64 | try { 65 | new MmapChunkAllocator(10240, dirs, new Configuration()); 66 | Assert.fail("Should fail because of delivering a file"); 67 | } catch (Exception e) { 68 | Assert.assertTrue(e instanceof FlinkRuntimeException); 69 | } 70 | 71 | Assert.assertFalse(new File(dirs[0], MmapChunkAllocator.MMAP_DIR).exists()); 72 | Assert.assertFalse(new File(dirs[1], MmapChunkAllocator.MMAP_DIR).exists()); 73 | Assert.assertTrue(dirs[2].exists()); 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /flink-spillable-benchmark/src/main/java/org/apache/flink/spillable/benchmark/JobConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.spillable.benchmark; 20 | 21 | import org.apache.flink.api.java.utils.ParameterTool; 22 | import org.apache.flink.configuration.ConfigOption; 23 | import org.apache.flink.configuration.ConfigOptions; 24 | import org.apache.flink.configuration.Configuration; 25 | 26 | /** 27 | * Configurations for the job. 28 | */ 29 | public class JobConfig { 30 | 31 | public static final ConfigOption JOB_NAME = ConfigOptions 32 | .key("jobName") 33 | .defaultValue("WordCount") 34 | .withDescription("Job name"); 35 | 36 | // word source options ====================================== 37 | 38 | public static final ConfigOption WORD_NUMBER = ConfigOptions 39 | .key("wordNumber") 40 | .defaultValue(100000) 41 | .withDescription("Number of different words which will influence state size."); 42 | 43 | public static final ConfigOption WORD_LENGTH = ConfigOptions 44 | .key("wordLength") 45 | .defaultValue(16) 46 | .withDescription("Length of word which will influence state size."); 47 | 48 | public static final ConfigOption WORD_RATE = ConfigOptions 49 | .key("wordRate") 50 | .defaultValue(1000000) 51 | .withDescription("Rate to emit words"); 52 | 53 | public static final ConfigOption CHECKPOINT_INTERVAL = ConfigOptions 54 | .key("checkpointInterval") 55 | .defaultValue(Long.MAX_VALUE) 56 | .withDescription("Checkpoint interval in milliseconds, and default is Long.MAX_VALUE which" + 57 | "means checkpoint is disable."); 58 | 59 | public static Configuration getConfiguration(ParameterTool params) { 60 | Configuration configuration = new Configuration(); 61 | 62 | 63 | // command line configurations have high priority 64 | configuration.addAll(params.getConfiguration()); 65 | 66 | return configuration; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/estimate/StateMemoryEstimatorFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap.estimate; 20 | 21 | import org.apache.flink.api.common.state.StateDescriptor; 22 | import org.apache.flink.api.common.typeutils.TypeSerializer; 23 | import org.apache.flink.api.common.typeutils.base.ListSerializer; 24 | import org.apache.flink.api.common.typeutils.base.MapSerializer; 25 | import org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo; 26 | 27 | /** 28 | * Factory to create {@link StateMemoryEstimator}. 29 | */ 30 | public class StateMemoryEstimatorFactory { 31 | 32 | @SuppressWarnings("unchecked") 33 | public static SampleStateMemoryEstimator createSampleEstimator( 34 | TypeSerializer keySerializer, 35 | RegisteredKeyValueStateBackendMetaInfo metaInfo, 36 | int estimateSampleCount) { 37 | StateDescriptor.Type stateType = metaInfo.getStateType(); 38 | StateMemoryEstimator stateMemoryEstimator; 39 | switch (stateType) { 40 | case VALUE: 41 | case FOLDING: 42 | case REDUCING: 43 | case AGGREGATING: 44 | stateMemoryEstimator = new ValueStateMemoryEstimator<>( 45 | keySerializer, metaInfo.getNamespaceSerializer(), metaInfo.getStateSerializer()); 46 | break; 47 | case MAP: 48 | stateMemoryEstimator = new MapStateMemoryEstimator<>( 49 | keySerializer, metaInfo.getNamespaceSerializer(), (MapSerializer) metaInfo.getStateSerializer()); 50 | break; 51 | case LIST: 52 | stateMemoryEstimator = new ListStateMemoryEstimator<>( 53 | keySerializer, metaInfo.getNamespaceSerializer(), (ListSerializer) metaInfo.getStateSerializer()); 54 | break; 55 | default: 56 | throw new UnsupportedOperationException("Unknown state type " + stateType.name()); 57 | } 58 | 59 | return new SampleStateMemoryEstimator<>((StateMemoryEstimator) stateMemoryEstimator, estimateSampleCount); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/SkipListValueSerializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap; 20 | 21 | import org.apache.flink.api.common.typeutils.TypeSerializer; 22 | import org.apache.flink.core.memory.DataInputViewStreamWrapper; 23 | import org.apache.flink.core.memory.DataOutputSerializer; 24 | import org.apache.flink.core.memory.MemorySegment; 25 | import org.apache.flink.core.memory.MemorySegmentInputStreamWithPos; 26 | 27 | import java.io.IOException; 28 | 29 | /** 30 | * Serializer/deserializer used for conversion between state and skip list value. 31 | * It is not thread safe. 32 | * 33 | * @param type of state. 34 | */ 35 | class SkipListValueSerializer { 36 | 37 | private final TypeSerializer stateSerializer; 38 | 39 | /** The reusable output serialization buffer. */ 40 | private final DataOutputSerializer dos; 41 | 42 | SkipListValueSerializer(TypeSerializer stateSerializer) { 43 | this.stateSerializer = stateSerializer; 44 | this.dos = new DataOutputSerializer(16); 45 | } 46 | 47 | byte[] serialize(S state) { 48 | try { 49 | stateSerializer.serialize(state, dos); 50 | } catch (IOException e) { 51 | throw new RuntimeException("serialize key and namespace failed", e); 52 | } 53 | byte[] ret = dos.getCopyOfBuffer(); 54 | dos.clear(); 55 | return ret; 56 | } 57 | 58 | /** 59 | * Deserialize the state from the byte buffer which stores skip list value. 60 | * 61 | * @param memorySegment the memory segment which stores the skip list value. 62 | * @param offset the start position of the skip list value in the byte buffer. 63 | * @param len length of the skip list value. 64 | */ 65 | S deserializeState(MemorySegment memorySegment, int offset, int len) { 66 | final MemorySegmentInputStreamWithPos src = new MemorySegmentInputStreamWithPos(memorySegment, offset, len); 67 | final DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(src); 68 | 69 | try { 70 | return stateSerializer.deserialize(in); 71 | } catch (IOException e) { 72 | throw new RuntimeException("deserialize state failed", e); 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /tools/maven/suppressions.xml: -------------------------------------------------------------------------------- 1 | 2 | 20 | 21 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 38 | 39 | 42 | 43 | 46 | 47 | 50 | 53 | 56 | 57 | 60 | 61 | 64 | 65 | -------------------------------------------------------------------------------- /flink-spillable-benchmark/src/main/java/org/apache/flink/spillable/benchmark/ThrottledIterator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | 20 | package org.apache.flink.spillable.benchmark; 21 | 22 | import java.io.Serializable; 23 | import java.util.Iterator; 24 | 25 | import static java.util.Objects.requireNonNull; 26 | 27 | /** 28 | * A variant of the collection source (emits a sequence of elements as a stream) 29 | * that supports throttling the emission rate. 30 | */ 31 | public class ThrottledIterator implements Iterator, Serializable { 32 | 33 | private static final long serialVersionUID = 1L; 34 | 35 | private final Iterator source; 36 | 37 | private final long sleepBatchSize; 38 | private final long sleepBatchTime; 39 | 40 | private long lastBatchCheckTime; 41 | private long num; 42 | 43 | public ThrottledIterator(Iterator source, long elementsPerSecond) { 44 | this.source = requireNonNull(source); 45 | 46 | if (elementsPerSecond >= 100) { 47 | // how many elements would we emit per 50ms 48 | this.sleepBatchSize = elementsPerSecond / 20; 49 | this.sleepBatchTime = 50; 50 | } 51 | else if (elementsPerSecond >= 1) { 52 | // how long does element take 53 | this.sleepBatchSize = 1; 54 | this.sleepBatchTime = 1000 / elementsPerSecond; 55 | } 56 | else { 57 | throw new IllegalArgumentException("'elements per second' must be positive and not zero"); 58 | } 59 | } 60 | 61 | @Override 62 | public boolean hasNext() { 63 | return source.hasNext(); 64 | } 65 | 66 | @Override 67 | public T next() { 68 | // delay if necessary 69 | if (lastBatchCheckTime > 0) { 70 | if (++num >= sleepBatchSize) { 71 | num = 0; 72 | 73 | final long now = System.currentTimeMillis(); 74 | final long elapsed = now - lastBatchCheckTime; 75 | if (elapsed < sleepBatchTime) { 76 | try { 77 | Thread.sleep(sleepBatchTime - elapsed); 78 | } catch (InterruptedException e) { 79 | // restore interrupt flag and proceed 80 | Thread.currentThread().interrupt(); 81 | } 82 | } 83 | lastBatchCheckTime = System.currentTimeMillis(); 84 | } 85 | } else { 86 | lastBatchCheckTime = System.currentTimeMillis(); 87 | } 88 | 89 | return source.next(); 90 | } 91 | 92 | @Override 93 | public void remove() { 94 | throw new UnsupportedOperationException(); 95 | } 96 | } 97 | 98 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/estimate/ListStateMemoryEstimator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap.estimate; 20 | 21 | import org.apache.flink.api.common.typeutils.TypeSerializer; 22 | import org.apache.flink.api.common.typeutils.base.ListSerializer; 23 | 24 | import java.util.ArrayList; 25 | import java.util.List; 26 | 27 | /** 28 | * Estimates memory usage of {@link org.apache.flink.api.common.state.ListState}. 29 | * 30 | * @param type of key 31 | * @param type of namespace 32 | */ 33 | public class ListStateMemoryEstimator extends AbstractStateMemoryEstimator { 34 | 35 | private static final long LIST_SHADOW_SIZE; 36 | 37 | static { 38 | List list = new ArrayList<>(); 39 | LIST_SHADOW_SIZE = RamUsageEstimator.shallowSizeOf(list); 40 | } 41 | 42 | private boolean isElementFix; 43 | private long elementSize; 44 | 45 | public ListStateMemoryEstimator( 46 | TypeSerializer keySerializer, 47 | TypeSerializer namespaceSerializer, 48 | ListSerializer listSerializer) { 49 | super(keySerializer, namespaceSerializer); 50 | this.isElementFix = listSerializer.getElementSerializer().getLength() != -1; 51 | this.elementSize = -1; 52 | } 53 | 54 | @Override 55 | protected long getStateSize(List state) { 56 | return getDataStructureSize(state) + getListSize(state); 57 | } 58 | 59 | private long getDataStructureSize(List list) { 60 | assert list instanceof ArrayList; 61 | int size = list.size(); 62 | long arraySize = RamUsageEstimator.alignObjectSize( 63 | (long) RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + 64 | (long) RamUsageEstimator.NUM_BYTES_OBJECT_REF * size); 65 | 66 | return LIST_SHADOW_SIZE + arraySize; 67 | } 68 | 69 | private long getListSize(List list) { 70 | if (list.isEmpty()) { 71 | return 0; 72 | } 73 | 74 | long totalSize = 0; 75 | if (isElementFix) { 76 | totalSize += getElementSize(list.get(0)) * list.size(); 77 | } else { 78 | for (Object element : list) { 79 | totalSize += getElementSize(element); 80 | } 81 | } 82 | 83 | return totalSize; 84 | } 85 | 86 | private long getElementSize(Object element) { 87 | if (!isElementFix || elementSize == -1) { 88 | this.elementSize = RamUsageEstimator.sizeOf(element); 89 | } 90 | 91 | return elementSize; 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/SpillableStateTableSnapshot.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * * Licensed to the Apache Software Foundation (ASF) under one 4 | * * or more contributor license agreements. See the NOTICE file 5 | * * distributed with this work for additional information 6 | * * regarding copyright ownership. The ASF licenses this file 7 | * * to you under the Apache License, Version 2.0 (the 8 | * * "License"); you may not use this file except in compliance 9 | * * with the License. You may obtain a copy of the License at 10 | * * 11 | * * http://www.apache.org/licenses/LICENSE-2.0 12 | * * 13 | * * Unless required by applicable law or agreed to in writing, software 14 | * * distributed under the License is distributed on an "AS IS" BASIS, 15 | * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | * * See the License for the specific language governing permissions and 17 | * * limitations under the License. 18 | * 19 | */ 20 | 21 | package org.apache.flink.runtime.state.heap; 22 | 23 | import org.apache.flink.api.common.typeutils.TypeSerializer; 24 | import org.apache.flink.runtime.state.StateSnapshotTransformer; 25 | 26 | import javax.annotation.Nonnull; 27 | 28 | import java.util.List; 29 | 30 | /** 31 | * This class represents the snapshot of a {@link SpillableStateTableImpl} and has a role in operator state checkpointing. 32 | * This class is also responsible for writing the state in the process of checkpointing. 33 | * 34 | * @param type of key 35 | * @param type of namespace 36 | * @param type of state 37 | */ 38 | public class SpillableStateTableSnapshot extends AbstractStateTableSnapshot { 39 | 40 | /** 41 | * The offset to the contiguous key groups. 42 | */ 43 | private final int keyGroupOffset; 44 | 45 | /** 46 | * Snapshots of state partitioned by key-group. 47 | */ 48 | @Nonnull 49 | private final List>> stateMapSnapshots; 50 | 51 | /** 52 | * Creates a new {@link CopyOnWriteSkipListStateMapSnapshot}. 53 | * 54 | * @param owningStateTable the {@link SpillableStateTableImpl} for which this object represents a snapshot. 55 | */ 56 | SpillableStateTableSnapshot( 57 | SpillableStateTableImpl owningStateTable, 58 | TypeSerializer localKeySerializer, 59 | TypeSerializer localNamespaceSerializer, 60 | TypeSerializer localStateSerializer, 61 | StateSnapshotTransformer stateSnapshotTransformer) { 62 | super(owningStateTable, 63 | localKeySerializer, 64 | localNamespaceSerializer, 65 | localStateSerializer, 66 | stateSnapshotTransformer); 67 | 68 | this.keyGroupOffset = owningStateTable.getKeyGroupOffset(); 69 | this.stateMapSnapshots = owningStateTable.getStateMapSnapshotList(); 70 | } 71 | 72 | @Override 73 | protected StateMapSnapshot> getStateMapSnapshotForKeyGroup(int keyGroup) { 74 | int indexOffset = keyGroup - keyGroupOffset; 75 | StateMapSnapshot> stateMapSnapshot = null; 76 | if (indexOffset >= 0 && indexOffset < stateMapSnapshots.size()) { 77 | stateMapSnapshot = stateMapSnapshots.get(indexOffset); 78 | } 79 | 80 | return stateMapSnapshot; 81 | } 82 | 83 | @Override 84 | public void release() { 85 | for (StateMapSnapshot snapshot : stateMapSnapshots) { 86 | snapshot.release(); 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/SkipListKeyComparator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap; 20 | 21 | import org.apache.flink.core.memory.MemorySegment; 22 | 23 | /** 24 | * Comparator used for skip list key. 25 | */ 26 | class SkipListKeyComparator { 27 | 28 | /** 29 | * Compares for order. Returns a negative integer, zero, or a positive integer 30 | * as the first node is less than, equal to, or greater than the second. 31 | * 32 | * @param left left skip list key's ByteBuffer 33 | * @param leftOffset left skip list key's ByteBuffer's offset 34 | * @param right right skip list key's ByteBuffer 35 | * @param rightOffset right skip list key's ByteBuffer's offset 36 | * @return An integer result of the comparison. 37 | */ 38 | static int compareTo(MemorySegment left, int leftOffset, MemorySegment right, int rightOffset) { 39 | // compare namespace 40 | int leftNamespaceLen = left.getInt(leftOffset); 41 | int rightNamespaceLen = right.getInt(rightOffset); 42 | 43 | int c = left.compare(right, leftOffset + Integer.BYTES, rightOffset + Integer.BYTES, 44 | leftNamespaceLen, rightNamespaceLen); 45 | 46 | if (c != 0) { 47 | return c; 48 | } 49 | 50 | // compare key 51 | int leftKeyOffset = leftOffset + Integer.BYTES + leftNamespaceLen; 52 | int rightKeyOffset = rightOffset + Integer.BYTES + rightNamespaceLen; 53 | int leftKeyLen = left.getInt(leftKeyOffset); 54 | int rightKeyLen = right.getInt(rightKeyOffset); 55 | 56 | return left.compare(right, leftKeyOffset + Integer.BYTES, rightKeyOffset + Integer.BYTES, 57 | leftKeyLen, rightKeyLen); 58 | } 59 | 60 | /** 61 | * Compares the namespace in the memory segment with the namespace in the node . 62 | * Returns a negative integer, zero, or a positive integer as the first node is 63 | * less than, equal to, or greater than the second. 64 | * 65 | * @param namespaceSegment memory segment to store the namespace. 66 | * @param namespaceOffset offset of namespace in the memory segment. 67 | * @param namespaceLen length of namespace. 68 | * @param nodeSegment memory segment to store the node key. 69 | * @param nodeKeyOffset offset of node key in the memory segment. 70 | * @return An integer result of the comparison. 71 | */ 72 | static int compareNamespaceAndNode( 73 | MemorySegment namespaceSegment, int namespaceOffset, int namespaceLen, 74 | MemorySegment nodeSegment, int nodeKeyOffset) { 75 | 76 | int nodeNamespaceLen = nodeSegment.getInt(nodeKeyOffset); 77 | return namespaceSegment.compare(nodeSegment, namespaceOffset, nodeKeyOffset + Integer.BYTES, 78 | namespaceLen, nodeNamespaceLen); 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/CheckpointManagerImpl.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap; 20 | 21 | import org.apache.flink.annotation.VisibleForTesting; 22 | import org.apache.flink.runtime.state.KeyedStateHandle; 23 | import org.apache.flink.runtime.state.SnapshotResult; 24 | 25 | import org.slf4j.Logger; 26 | import org.slf4j.LoggerFactory; 27 | 28 | import java.util.Map; 29 | import java.util.concurrent.ConcurrentHashMap; 30 | import java.util.concurrent.RunnableFuture; 31 | 32 | /** 33 | * Implementation of {@link CheckpointManager}. 34 | */ 35 | public class CheckpointManagerImpl implements CheckpointManager { 36 | 37 | private static final Logger LOG = LoggerFactory.getLogger(CheckpointManagerImpl.class); 38 | 39 | private Map>> runningCheckpoints; 40 | 41 | public CheckpointManagerImpl() { 42 | this.runningCheckpoints = new ConcurrentHashMap<>(); 43 | } 44 | 45 | @Override 46 | public boolean registerCheckpoint( 47 | long checkpointId, 48 | RunnableFuture> runnableFuture) { 49 | if (runningCheckpoints.containsKey(checkpointId)) { 50 | LOG.warn("Checkpoint {} has been registered.", checkpointId); 51 | return false; 52 | } else { 53 | runningCheckpoints.put(checkpointId, runnableFuture); 54 | LOG.debug("Register checkpoint {}.", checkpointId); 55 | return true; 56 | } 57 | } 58 | 59 | @Override 60 | public boolean unRegisterCheckpoint(long checkpointId) { 61 | if (runningCheckpoints.remove(checkpointId) != null) { 62 | LOG.debug("Unregister checkpoint {}.", checkpointId); 63 | return true; 64 | } else { 65 | LOG.warn("Try to unregister a non-exist checkpoint {}.", checkpointId); 66 | return false; 67 | } 68 | } 69 | 70 | @Override 71 | public boolean cancelCheckpoint(long checkpointId) { 72 | RunnableFuture> runnableFuture = 73 | runningCheckpoints.get(checkpointId); 74 | if (runnableFuture != null) { 75 | runnableFuture.cancel(true); 76 | LOG.info("Cancel checkpoint {}.", checkpointId); 77 | return true; 78 | } else { 79 | LOG.warn("Try to cancel a non-exist checkpoint {}.", checkpointId); 80 | return false; 81 | } 82 | } 83 | 84 | @Override 85 | public void cancelAllCheckpoints() { 86 | for (Map.Entry>> entry : runningCheckpoints.entrySet()) { 87 | entry.getValue().cancel(true); 88 | LOG.info("Cancel checkpoint {}.", entry.getKey()); 89 | } 90 | } 91 | 92 | @VisibleForTesting 93 | Map>> getRunningCheckpoints() { 94 | return runningCheckpoints; 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/SpillableValueState.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap; 20 | 21 | import org.apache.flink.api.common.state.State; 22 | import org.apache.flink.api.common.state.StateDescriptor; 23 | import org.apache.flink.api.common.state.ValueState; 24 | import org.apache.flink.api.common.typeutils.TypeSerializer; 25 | import org.apache.flink.runtime.state.internal.InternalValueState; 26 | 27 | /** 28 | * Heap-backed partitioned {@link ValueState} that is snapshotted into files. 29 | * 30 | * @param The type of the key. 31 | * @param The type of the namespace. 32 | * @param The type of the value. 33 | */ 34 | class SpillableValueState 35 | extends AbstractHeapState 36 | implements InternalValueState { 37 | 38 | /** 39 | * Creates a new key/value state for the given hash map of key/value pairs. 40 | * 41 | * @param stateTable The state table for which this state is associated to. 42 | * @param keySerializer The serializer for the keys. 43 | * @param valueSerializer The serializer for the state. 44 | * @param namespaceSerializer The serializer for the namespace. 45 | * @param defaultValue The default value for the state. 46 | */ 47 | private SpillableValueState( 48 | StateTable stateTable, 49 | TypeSerializer keySerializer, 50 | TypeSerializer valueSerializer, 51 | TypeSerializer namespaceSerializer, 52 | V defaultValue) { 53 | super(stateTable, keySerializer, valueSerializer, namespaceSerializer, defaultValue); 54 | } 55 | 56 | @Override 57 | public TypeSerializer getKeySerializer() { 58 | return keySerializer; 59 | } 60 | 61 | @Override 62 | public TypeSerializer getNamespaceSerializer() { 63 | return namespaceSerializer; 64 | } 65 | 66 | @Override 67 | public TypeSerializer getValueSerializer() { 68 | return valueSerializer; 69 | } 70 | 71 | @Override 72 | public V value() { 73 | final V result = stateTable.get(currentNamespace); 74 | 75 | if (result == null) { 76 | return getDefaultValue(); 77 | } 78 | 79 | return result; 80 | } 81 | 82 | @Override 83 | public void update(V value) { 84 | 85 | if (value == null) { 86 | clear(); 87 | return; 88 | } 89 | 90 | stateTable.put(currentNamespace, value); 91 | } 92 | 93 | @SuppressWarnings("unchecked") 94 | static IS create( 95 | StateDescriptor stateDesc, 96 | StateTable stateTable, 97 | TypeSerializer keySerializer) { 98 | return (IS) new SpillableValueState<>( 99 | stateTable, 100 | keySerializer, 101 | stateTable.getStateSerializer(), 102 | stateTable.getNamespaceSerializer(), 103 | stateDesc.getDefaultValue()); 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/SpillableStateTable.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap; 20 | 21 | import org.apache.flink.api.common.typeutils.TypeSerializer; 22 | import org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo; 23 | 24 | import java.io.Closeable; 25 | import java.util.Iterator; 26 | 27 | /** 28 | * Base class for spillable state table. 29 | */ 30 | public abstract class SpillableStateTable extends StateTable implements Closeable { 31 | 32 | public SpillableStateTable( 33 | InternalKeyContext keyContext, 34 | RegisteredKeyValueStateBackendMetaInfo metaInfo, 35 | TypeSerializer keySerializer) { 36 | super(keyContext, metaInfo, keySerializer); 37 | } 38 | 39 | /** 40 | * Update estimated memory size of state. 41 | */ 42 | public abstract void updateStateEstimate(N namespace, S state); 43 | 44 | /** 45 | * Return the estimated size of state. When {@param force} is true, 46 | * an estimation will be made if there hasn't been an estimation. 47 | */ 48 | public abstract long getStateEstimatedSize(boolean force); 49 | 50 | /** 51 | * Spill state in the given key group. 52 | */ 53 | public abstract void spillState(int keyGroupIndex); 54 | 55 | /** 56 | * Load state in the given key group. 57 | */ 58 | public abstract void loadState(int keyGroupIndex); 59 | 60 | /** 61 | * Returns an iterator over all state map metas. 62 | */ 63 | public abstract Iterator stateMapIterator(); 64 | 65 | /** 66 | * Meta of a {@link StateMap}. 67 | */ 68 | public static class StateMapMeta { 69 | 70 | private final SpillableStateTable stateTable; 71 | private final int keyGroupIndex; 72 | private final boolean isOnHeap; 73 | private final int size; 74 | private final long numRequests; 75 | /** Initialize lazily. -1 indicates uninitialized. */ 76 | private long estimatedMemorySize; 77 | 78 | public StateMapMeta( 79 | SpillableStateTable stateTable, 80 | int keyGroupIndex, 81 | boolean isOnHeap, 82 | int size, 83 | long numRequests) { 84 | this.stateTable = stateTable; 85 | this.keyGroupIndex = keyGroupIndex; 86 | this.isOnHeap = isOnHeap; 87 | this.size = size; 88 | this.numRequests = numRequests; 89 | this.estimatedMemorySize = -1; 90 | } 91 | 92 | public SpillableStateTable getStateTable() { 93 | return stateTable; 94 | } 95 | 96 | public boolean isOnHeap() { 97 | return isOnHeap; 98 | } 99 | 100 | public int getSize() { 101 | return size; 102 | } 103 | 104 | public int getKeyGroupIndex() { 105 | return keyGroupIndex; 106 | } 107 | 108 | public long getNumRequests() { 109 | return numRequests; 110 | } 111 | 112 | public long getEstimatedMemorySize() { 113 | return estimatedMemorySize; 114 | } 115 | 116 | public void setEstimatedMemorySize(long estimatedMemorySize) { 117 | this.estimatedMemorySize = estimatedMemorySize; 118 | } 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/OnHeapLevelIndexHeader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap; 20 | 21 | import org.apache.flink.annotation.VisibleForTesting; 22 | import org.apache.flink.util.Preconditions; 23 | 24 | import static org.apache.flink.runtime.state.heap.SkipListUtils.DEFAULT_LEVEL; 25 | import static org.apache.flink.runtime.state.heap.SkipListUtils.MAX_LEVEL; 26 | import static org.apache.flink.runtime.state.heap.SkipListUtils.NIL_NODE; 27 | 28 | /** 29 | * Implementation of {@link LevelIndexHeader} which stores index on heap. 30 | */ 31 | public class OnHeapLevelIndexHeader implements LevelIndexHeader { 32 | 33 | /** 34 | * The level index array where each position stores the next node id of the level. 35 | */ 36 | private volatile long[] levelIndex; 37 | 38 | /** 39 | * The topmost level currently. 40 | */ 41 | private volatile int topLevel; 42 | 43 | /** 44 | * Next node at level 0. 45 | */ 46 | private volatile long nextNode; 47 | 48 | OnHeapLevelIndexHeader() { 49 | this(DEFAULT_LEVEL); 50 | } 51 | 52 | private OnHeapLevelIndexHeader(int maxLevel) { 53 | Preconditions.checkArgument(maxLevel >= 1 && maxLevel <= MAX_LEVEL, 54 | "maxLevel(" + maxLevel + ") must be non-negative and no more than " + MAX_LEVEL); 55 | this.topLevel = 1; 56 | this.nextNode = NIL_NODE; 57 | this.levelIndex = new long[maxLevel]; 58 | initLevelIndex(levelIndex); 59 | } 60 | 61 | private void initLevelIndex(long[] levelIndex) { 62 | for (int i = 0; i < levelIndex.length; i++) { 63 | levelIndex[i] = NIL_NODE; 64 | } 65 | } 66 | 67 | @Override 68 | public int getLevel() { 69 | return topLevel; 70 | } 71 | 72 | @Override 73 | public void updateLevel(int level) { 74 | Preconditions.checkArgument(level >= 0 && level <= MAX_LEVEL, 75 | "level(" + level + ") must be non-negative and no more than " + MAX_LEVEL); 76 | Preconditions.checkArgument(level <= this.topLevel + 1, 77 | "top level " + topLevel + " must be updated level by level, but new level is " + level); 78 | 79 | if (levelIndex.length < level) { 80 | long[] newLevelIndex = new long[this.levelIndex.length * 2]; 81 | initLevelIndex(newLevelIndex); 82 | System.arraycopy(this.levelIndex, 0, newLevelIndex, 0, this.levelIndex.length); 83 | this.levelIndex = newLevelIndex; 84 | } 85 | 86 | if (topLevel < level) { 87 | topLevel = level; 88 | } 89 | } 90 | 91 | @Override 92 | public long getNextNode(int level) { 93 | Preconditions.checkArgument(level >= 0 && level <= topLevel, 94 | "invalid level " + level + " current top level is " + topLevel); 95 | 96 | if (level == 0) { 97 | return nextNode; 98 | } 99 | return levelIndex[level - 1]; 100 | } 101 | 102 | @Override 103 | public void updateNextNode(int level, long node) { 104 | Preconditions.checkArgument(level >= 0 && level <= topLevel, 105 | "invalid level " + level + " current top level is " + topLevel); 106 | 107 | if (level == 0) { 108 | nextNode = node; 109 | } else { 110 | levelIndex[level - 1] = node; 111 | } 112 | } 113 | 114 | @VisibleForTesting 115 | long[] getLevelIndex() { 116 | return levelIndex; 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/estimate/Constants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.flink.runtime.state.heap.estimate; 19 | 20 | import java.util.StringTokenizer; 21 | 22 | /** 23 | * Some useful constants. 24 | * Refer to https://github.com/apache/lucene-solr/blob/master/lucene/core/src/java/org/apache/lucene/util/Constants.java. 25 | */ 26 | public final class Constants { 27 | 28 | /** 29 | * Can't construct. 30 | */ 31 | private Constants() {} 32 | 33 | /** JVM vendor info. */ 34 | public static final String JVM_VENDOR = System.getProperty("java.vm.vendor"); 35 | public static final String JVM_VERSION = System.getProperty("java.vm.version"); 36 | public static final String JVM_NAME = System.getProperty("java.vm.name"); 37 | public static final String JVM_SPEC_VERSION = System.getProperty("java.specification.version"); 38 | 39 | /** The value of System.getProperty("java.version"). **/ 40 | public static final String JAVA_VERSION = System.getProperty("java.version"); 41 | 42 | /** The value of System.getProperty("os.name"). **/ 43 | public static final String OS_NAME = System.getProperty("os.name"); 44 | /** True iff running on Linux. */ 45 | public static final boolean LINUX = OS_NAME.startsWith("Linux"); 46 | /** True iff running on Windows. */ 47 | public static final boolean WINDOWS = OS_NAME.startsWith("Windows"); 48 | /** True iff running on SunOS. */ 49 | public static final boolean SUN_OS = OS_NAME.startsWith("SunOS"); 50 | /** True iff running on Mac OS X. */ 51 | public static final boolean MAC_OS_X = OS_NAME.startsWith("Mac OS X"); 52 | /** True iff running on FreeBSD. */ 53 | public static final boolean FREE_BSD = OS_NAME.startsWith("FreeBSD"); 54 | 55 | public static final String OS_ARCH = System.getProperty("os.arch"); 56 | public static final String OS_VERSION = System.getProperty("os.version"); 57 | public static final String JAVA_VENDOR = System.getProperty("java.vendor"); 58 | 59 | private static final int JVM_MAJOR_VERSION; 60 | private static final int JVM_MINOR_VERSION; 61 | 62 | /** True iff running on a 64bit JVM. */ 63 | public static final boolean JRE_IS_64BIT; 64 | 65 | static { 66 | final StringTokenizer st = new StringTokenizer(JVM_SPEC_VERSION, "."); 67 | JVM_MAJOR_VERSION = Integer.parseInt(st.nextToken()); 68 | if (st.hasMoreTokens()) { 69 | JVM_MINOR_VERSION = Integer.parseInt(st.nextToken()); 70 | } else { 71 | JVM_MINOR_VERSION = 0; 72 | } 73 | boolean is64Bit = false; 74 | String datamodel = null; 75 | try { 76 | datamodel = System.getProperty("sun.arch.data.model"); 77 | if (datamodel != null) { 78 | is64Bit = datamodel.contains("64"); 79 | } 80 | } catch (SecurityException ex) {} 81 | if (datamodel == null) { 82 | if (OS_ARCH != null && OS_ARCH.contains("64")) { 83 | is64Bit = true; 84 | } else { 85 | is64Bit = false; 86 | } 87 | } 88 | JRE_IS_64BIT = is64Bit; 89 | } 90 | 91 | public static final boolean JRE_IS_MINIMUM_JAVA8 = JVM_MAJOR_VERSION > 1 || (JVM_MAJOR_VERSION == 1 && JVM_MINOR_VERSION >= 8); 92 | public static final boolean JRE_IS_MINIMUM_JAVA9 = JVM_MAJOR_VERSION > 1 || (JVM_MAJOR_VERSION == 1 && JVM_MINOR_VERSION >= 9); 93 | public static final boolean JRE_IS_MINIMUM_JAVA11 = JVM_MAJOR_VERSION > 1 || (JVM_MAJOR_VERSION == 1 && JVM_MINOR_VERSION >= 11); 94 | } 95 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/space/BlockAllocator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap.space; 20 | 21 | import org.apache.flink.annotation.VisibleForTesting; 22 | 23 | import org.apache.commons.collections.map.LinkedMap; 24 | 25 | import java.util.concurrent.ConcurrentLinkedQueue; 26 | 27 | import static org.apache.flink.runtime.state.heap.space.SpaceConstants.NO_SPACE; 28 | 29 | /** 30 | * Allocates blocks with fixed size from {@link Bucket}. 31 | */ 32 | final class BlockAllocator { 33 | 34 | /** 35 | * Index of this block allocator. 36 | */ 37 | private final int blockAllocatorIndex; 38 | 39 | /** 40 | * Queue of free buckets. Allocator can apply a free bucket 41 | * from the queue, and returns a free bucket to the queue. 42 | */ 43 | private final ConcurrentLinkedQueue freeBucketsQueue; 44 | 45 | /** 46 | * Map of buckets applied by this allocator from queue. 47 | */ 48 | private LinkedMap appliedBuckets; 49 | 50 | /** 51 | * Map of buckets that have free blocks to allocate. 52 | */ 53 | private LinkedMap freeBuckets; 54 | 55 | BlockAllocator(int blockAllocatorIndex, ConcurrentLinkedQueue freeBucketsQueue) { 56 | this.blockAllocatorIndex = blockAllocatorIndex; 57 | this.freeBucketsQueue = freeBucketsQueue; 58 | this.appliedBuckets = new LinkedMap(); 59 | this.freeBuckets = new LinkedMap(); 60 | } 61 | 62 | /** 63 | * Allocates a block. 64 | * 65 | * @return offset of the block. 66 | */ 67 | synchronized int allocateBlock() { 68 | Bucket bucket = null; 69 | 70 | // 1. try to find a bucket that have free blocks 71 | if (freeBuckets.size() > 0) { 72 | bucket = (Bucket) freeBuckets.lastKey(); 73 | } 74 | 75 | // 2. try to apply a new bucket from queue 76 | if (bucket == null) { 77 | bucket = freeBucketsQueue.poll(); 78 | if (bucket != null) { 79 | addBucket(bucket); 80 | } 81 | } 82 | 83 | if (bucket == null) { 84 | return NO_SPACE; 85 | } 86 | 87 | int offset = bucket.allocate(); 88 | if (!bucket.hasFreeBlocks()) { 89 | freeBuckets.remove(bucket); 90 | } 91 | 92 | return offset; 93 | } 94 | 95 | /** 96 | * Frees a block from the given bucket. 97 | * 98 | * @param bucket bucket that block belongs to. 99 | * @param offset offset of block to free. 100 | */ 101 | synchronized void freeBlock(Bucket bucket, int offset) { 102 | bucket.free(offset); 103 | if (bucket.isFree()) { 104 | removeBucket(bucket); 105 | freeBucketsQueue.offer(bucket); 106 | } else if (!freeBuckets.containsKey(bucket)) { 107 | freeBuckets.put(bucket, bucket); 108 | } 109 | } 110 | 111 | private void addBucket(Bucket b) { 112 | b.reuseBucket(blockAllocatorIndex); 113 | appliedBuckets.put(b, b); 114 | freeBuckets.put(b, b); 115 | } 116 | 117 | private void removeBucket(Bucket b) { 118 | appliedBuckets.remove(b); 119 | freeBuckets.remove(b); 120 | } 121 | 122 | @VisibleForTesting 123 | int getBlockAllocatorIndex() { 124 | return blockAllocatorIndex; 125 | } 126 | 127 | @VisibleForTesting 128 | LinkedMap getAppliedBuckets() { 129 | return appliedBuckets; 130 | } 131 | 132 | @VisibleForTesting 133 | LinkedMap getFreeBuckets() { 134 | return freeBuckets; 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/estimate/MapStateMemoryEstimator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap.estimate; 20 | 21 | import org.apache.flink.api.common.state.MapState; 22 | import org.apache.flink.api.common.typeutils.TypeSerializer; 23 | import org.apache.flink.api.common.typeutils.base.MapSerializer; 24 | 25 | import java.util.HashMap; 26 | import java.util.Map; 27 | 28 | /** 29 | * Estimates memory usage of {@link MapState}. 30 | * 31 | * @param type of key 32 | * @param type of namespace 33 | */ 34 | public class MapStateMemoryEstimator extends AbstractStateMemoryEstimator { 35 | 36 | private static final long HASH_MAP_SHADOW_SIZE; 37 | private static final long ENTRY_SHADOW_SIZE; 38 | 39 | static { 40 | Map testMap = new HashMap<>(); 41 | testMap.put(1, 1); 42 | HASH_MAP_SHADOW_SIZE = RamUsageEstimator.shallowSizeOf(testMap); 43 | ENTRY_SHADOW_SIZE = RamUsageEstimator.shallowSizeOf(testMap.entrySet().iterator().next()); 44 | } 45 | 46 | private boolean isMapKeyFix; 47 | private long mapKeySize; 48 | 49 | private boolean isMapValueFix; 50 | private long mapValueSize; 51 | 52 | public MapStateMemoryEstimator( 53 | TypeSerializer keySerializer, 54 | TypeSerializer namespaceSerializer, 55 | MapSerializer mapSerializer) { 56 | super(keySerializer, namespaceSerializer); 57 | this.isMapKeyFix = mapSerializer.getKeySerializer().getLength() != -1; 58 | this.mapKeySize = -1; 59 | this.isMapValueFix = mapSerializer.getValueSerializer().getLength() != -1; 60 | this.mapValueSize = -1; 61 | } 62 | 63 | @Override 64 | protected long getStateSize(Map state) { 65 | return getDataStructureSize(state) + getMapKeyValueSize(state); 66 | } 67 | 68 | /** 69 | * Approximate size of {@link HashMap} excluding keys and values. 70 | */ 71 | private long getDataStructureSize(Map map) { 72 | assert map instanceof HashMap; 73 | int size = map.size(); 74 | long arraySize = RamUsageEstimator.alignObjectSize( 75 | (long) RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + 76 | (long) RamUsageEstimator.NUM_BYTES_OBJECT_REF * size); 77 | 78 | return HASH_MAP_SHADOW_SIZE + ENTRY_SHADOW_SIZE * size + arraySize; 79 | } 80 | 81 | private long getMapKeyValueSize(Map state) { 82 | if (state.size() == 0) { 83 | return 0; 84 | } 85 | 86 | long totalSize = 0; 87 | // if map key or value is not fixed, iterates all entries 88 | if (!isMapKeyFix || !isMapValueFix) { 89 | for (Object object : state.entrySet()) { 90 | Map.Entry entry = (Map.Entry) object; 91 | totalSize += getMapKeySize(entry.getKey()) + getMapValueSize(entry.getValue()); 92 | } 93 | } else { 94 | Map.Entry entry = (Map.Entry) (state.entrySet().iterator().next()); 95 | totalSize = state.size() * (getMapKeySize(entry.getKey()) + getMapValueSize(entry.getValue())); 96 | } 97 | 98 | return totalSize; 99 | } 100 | 101 | private long getMapKeySize(Object mapKey) { 102 | if (!isMapKeyFix || mapKeySize == -1) { 103 | this.mapKeySize = RamUsageEstimator.sizeOf(mapKey); 104 | } 105 | 106 | return mapKeySize; 107 | } 108 | 109 | private long getMapValueSize(Object mapValue) { 110 | if (!isMapValueFix || mapValueSize == -1) { 111 | this.mapValueSize = RamUsageEstimator.sizeOf(mapValue); 112 | } 113 | 114 | return mapValueSize; 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 21 | 24 | 25 | 4.0.0 26 | 27 | 28 | org.apache.flink 29 | flink-spillable-statebackend 30 | 1.0-SNAPSHOT 31 | 32 | 33 | flink-statebackend-heap-spillable 34 | flink-statebackend-heap-spillable 35 | jar 36 | ${flink.version} 37 | 38 | 39 | 4.12 40 | 2.21.0 41 | 2.0.4 42 | 1.3 43 | 44 | 45 | 46 | 47 | 48 | 49 | org.apache.flink 50 | flink-runtime_${scala.binary.version} 51 | ${flink.version} 52 | provided 53 | 54 | 55 | 56 | 57 | 58 | org.apache.flink 59 | flink-test-utils-junit 60 | ${flink.version} 61 | test 62 | 63 | 64 | 65 | org.apache.flink 66 | flink-runtime_${scala.binary.version} 67 | ${flink.version} 68 | test-jar 69 | test 70 | 71 | 72 | 73 | org.apache.flink 74 | flink-core 75 | ${flink.version} 76 | test-jar 77 | test 78 | 79 | 80 | 81 | org.apache.flink 82 | flink-streaming-java_${scala.binary.version} 83 | ${flink.version} 84 | test-jar 85 | test 86 | 87 | 88 | 89 | junit 90 | junit 91 | ${junit.version} 92 | jar 93 | test 94 | 95 | 96 | 97 | org.mockito 98 | mockito-core 99 | ${mockito.version} 100 | jar 101 | test 102 | 103 | 104 | 105 | org.hamcrest 106 | hamcrest-all 107 | ${hamcrest.version} 108 | jar 109 | test 110 | 111 | 112 | 113 | org.powermock 114 | powermock-module-junit4 115 | ${powermock.version} 116 | jar 117 | test 118 | 119 | 120 | 121 | org.powermock 122 | powermock-api-mockito2 123 | ${powermock.version} 124 | jar 125 | test 126 | 127 | 128 | org.mockito 129 | mockito-core 130 | 131 | 132 | 133 | 134 | 135 | -------------------------------------------------------------------------------- /flink-spillable-benchmark/src/main/java/org/apache/flink/spillable/benchmark/WordSource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.spillable.benchmark; 20 | 21 | import org.apache.flink.api.java.tuple.Tuple2; 22 | import org.apache.flink.configuration.Configuration; 23 | import org.apache.flink.streaming.api.datastream.DataStreamSource; 24 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 25 | import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction; 26 | 27 | import java.io.Serializable; 28 | import java.util.Iterator; 29 | import java.util.Random; 30 | 31 | /** 32 | * Source to emit words at the given rate. 33 | */ 34 | public class WordSource extends RichParallelSourceFunction> { 35 | 36 | private static final long serialVersionUID = 1L; 37 | 38 | private final int wordLen; 39 | 40 | private final int wordNum; 41 | 42 | private final long wordRate; 43 | 44 | private transient ThrottledIterator throttledIterator; 45 | 46 | private transient char[] fatArray; 47 | 48 | private transient int emitNumber; 49 | 50 | private transient volatile boolean isRunning; 51 | 52 | public WordSource(int wordNum, long wordRate, int wordLen) { 53 | this.wordLen = wordLen; 54 | this.wordNum = wordNum; 55 | this.wordRate = wordRate; 56 | } 57 | 58 | @Override 59 | public void open(Configuration parameters) throws Exception { 60 | this.isRunning = true; 61 | this.emitNumber = 0; 62 | 63 | Iterator numberSourceIterator = new NumberSourceIterator(wordNum, System.currentTimeMillis()); 64 | this.throttledIterator = new ThrottledIterator<>(numberSourceIterator, wordRate); 65 | 66 | this.fatArray = new char[wordLen]; 67 | Random random = new Random(0); 68 | for (int i = 0; i < fatArray.length; i++) { 69 | fatArray[i] = (char) random.nextInt(); 70 | } 71 | } 72 | 73 | @Override 74 | public void run(SourceContext> sourceContext) throws Exception { 75 | while (isRunning) { 76 | Integer number; 77 | // first emit all keys 78 | if (emitNumber < wordNum) { 79 | number = emitNumber++; 80 | } else { 81 | number = throttledIterator.next(); 82 | } 83 | sourceContext.collect(Tuple2.of(covertToString(number), System.currentTimeMillis())); 84 | } 85 | } 86 | 87 | @Override 88 | public void cancel() { 89 | isRunning = false; 90 | } 91 | 92 | @Override 93 | public void close() { 94 | isRunning = false; 95 | } 96 | 97 | public static DataStreamSource> getSource(StreamExecutionEnvironment env, long wordRate, int wordNum, int wordLen) { 98 | return env.addSource(new WordSource(wordNum, wordRate, wordLen)); 99 | } 100 | 101 | private String covertToString(int number) { 102 | String a = String.valueOf(number); 103 | StringBuilder builder = new StringBuilder(wordLen); 104 | builder.append(a); 105 | builder.append(fatArray, 0, wordLen - a.length()); 106 | return builder.toString(); 107 | } 108 | 109 | // ------------------------------------------------------------------------ 110 | // Number generator 111 | // ------------------------------------------------------------------------ 112 | 113 | static class NumberSourceIterator implements Iterator, Serializable { 114 | private final int largest; 115 | private final Random rnd; 116 | 117 | public NumberSourceIterator(int largest, long seed) { 118 | this.largest = largest; 119 | this.rnd = new Random(seed); 120 | } 121 | 122 | @Override 123 | public boolean hasNext() { 124 | return true; 125 | } 126 | 127 | @Override 128 | public Integer next() { 129 | Integer value = rnd.nextInt(largest + 1); 130 | return value; 131 | } 132 | 133 | @Override 134 | public void remove() { 135 | throw new UnsupportedOperationException(); 136 | } 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /flink-spillable-benchmark/src/main/java/org/apache/flink/spillable/benchmark/WordCount.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.spillable.benchmark; 20 | 21 | import org.apache.flink.api.common.functions.RichFlatMapFunction; 22 | import org.apache.flink.api.common.state.ValueState; 23 | import org.apache.flink.api.common.state.ValueStateDescriptor; 24 | import org.apache.flink.api.common.typeinfo.TypeHint; 25 | import org.apache.flink.api.common.typeinfo.TypeInformation; 26 | import org.apache.flink.api.java.tuple.Tuple2; 27 | import org.apache.flink.api.java.utils.ParameterTool; 28 | import org.apache.flink.configuration.Configuration; 29 | import org.apache.flink.streaming.api.CheckpointingMode; 30 | import org.apache.flink.streaming.api.datastream.DataStream; 31 | import org.apache.flink.streaming.api.environment.CheckpointConfig; 32 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 33 | import org.apache.flink.streaming.api.functions.sink.DiscardingSink; 34 | import org.apache.flink.util.Collector; 35 | 36 | import java.io.IOException; 37 | 38 | import static org.apache.flink.spillable.benchmark.JobConfig.CHECKPOINT_INTERVAL; 39 | import static org.apache.flink.spillable.benchmark.JobConfig.JOB_NAME; 40 | import static org.apache.flink.spillable.benchmark.JobConfig.WORD_LENGTH; 41 | import static org.apache.flink.spillable.benchmark.JobConfig.WORD_NUMBER; 42 | import static org.apache.flink.spillable.benchmark.JobConfig.WORD_RATE; 43 | import static org.apache.flink.spillable.benchmark.JobConfig.getConfiguration; 44 | 45 | /** 46 | * A word count job. 47 | */ 48 | public class WordCount { 49 | 50 | public static void main(String[] args) throws Exception { 51 | 52 | final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); 53 | 54 | ParameterTool params = ParameterTool.fromArgs(args); 55 | Configuration configuration = getConfiguration(params); 56 | 57 | env.getConfig().setGlobalJobParameters(configuration); 58 | env.setParallelism(1); 59 | env.disableOperatorChaining(); 60 | 61 | String jobName = configuration.getString(JOB_NAME); 62 | 63 | env.enableCheckpointing(configuration.getLong(CHECKPOINT_INTERVAL), CheckpointingMode.EXACTLY_ONCE); 64 | env.getCheckpointConfig().enableExternalizedCheckpoints( 65 | CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION); 66 | 67 | // configure source 68 | int wordNumber = configuration.getInteger(WORD_NUMBER); 69 | int wordLength = configuration.getInteger(WORD_LENGTH); 70 | int wordRate = configuration.getInteger(WORD_RATE); 71 | 72 | DataStream> source = WordSource.getSource(env, wordRate, wordNumber, wordLength) 73 | .slotSharingGroup("src"); 74 | 75 | DataStream> mapper = source.keyBy(0) 76 | .flatMap(new Counter()) 77 | .slotSharingGroup("map"); 78 | 79 | mapper.addSink(new DiscardingSink<>()) 80 | .slotSharingGroup("sink"); 81 | 82 | env.execute(jobName); 83 | } 84 | 85 | /** 86 | * Write and read mixed mapper. 87 | */ 88 | public static class Counter extends RichFlatMapFunction, Tuple2> { 89 | 90 | private transient ValueState wordCounter; 91 | 92 | public Counter() { 93 | } 94 | 95 | @Override 96 | public void flatMap(Tuple2 in, Collector> out) throws IOException { 97 | Integer currentValue = wordCounter.value(); 98 | 99 | currentValue = currentValue == null ? 1 : currentValue + 1; 100 | wordCounter.update(currentValue); 101 | 102 | out.collect(Tuple2.of(in.f0, currentValue)); 103 | } 104 | 105 | @Override 106 | public void open(Configuration config) { 107 | ValueStateDescriptor descriptor = 108 | new ValueStateDescriptor<>( 109 | "wc", 110 | TypeInformation.of(new TypeHint(){})); 111 | wordCounter = getRuntimeContext().getState(descriptor); 112 | } 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/test/java/org/apache/flink/runtime/state/heap/SkipListSerializerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * * Licensed to the Apache Software Foundation (ASF) under one 4 | * * or more contributor license agreements. See the NOTICE file 5 | * * distributed with this work for additional information 6 | * * regarding copyright ownership. The ASF licenses this file 7 | * * to you under the Apache License, Version 2.0 (the 8 | * * "License"); you may not use this file except in compliance 9 | * * with the License. You may obtain a copy of the License at 10 | * * 11 | * * http://www.apache.org/licenses/LICENSE-2.0 12 | * * 13 | * * Unless required by applicable law or agreed to in writing, software 14 | * * distributed under the License is distributed on an "AS IS" BASIS, 15 | * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | * * See the License for the specific language governing permissions and 17 | * * limitations under the License. 18 | * 19 | */ 20 | 21 | package org.apache.flink.runtime.state.heap; 22 | 23 | import org.apache.flink.api.common.typeutils.TypeSerializer; 24 | import org.apache.flink.api.common.typeutils.base.StringSerializer; 25 | import org.apache.flink.api.java.tuple.Tuple2; 26 | import org.apache.flink.core.memory.ByteArrayInputStreamWithPos; 27 | import org.apache.flink.core.memory.DataInputViewStreamWrapper; 28 | import org.apache.flink.core.memory.MemorySegment; 29 | import org.apache.flink.core.memory.MemorySegmentFactory; 30 | import org.apache.flink.util.TestLogger; 31 | 32 | import org.junit.Test; 33 | 34 | import java.io.IOException; 35 | 36 | import static org.junit.Assert.assertEquals; 37 | 38 | /** 39 | * Tests for {@link SkipListKeySerializer}. 40 | */ 41 | public class SkipListSerializerTest extends TestLogger { 42 | private static final TypeSerializer keySerializer = StringSerializer.INSTANCE; 43 | private static final TypeSerializer namespaceSerializer = StringSerializer.INSTANCE; 44 | private static final SkipListKeySerializer skipListKeySerializer = 45 | new SkipListKeySerializer<>(keySerializer, namespaceSerializer); 46 | private static final TypeSerializer stateSerializer = StringSerializer.INSTANCE; 47 | private static final SkipListValueSerializer skipListValueSerializer = 48 | new SkipListValueSerializer<>(stateSerializer); 49 | 50 | @Test 51 | public void testSkipListKeySerializerBasicOp() throws IOException { 52 | testSkipListKeySerializer(0); 53 | } 54 | 55 | @Test 56 | public void testSkipListKeySerializerStateless() throws IOException { 57 | for (int i = 0; i < 10; i++) { 58 | testSkipListKeySerializer(i); 59 | } 60 | } 61 | 62 | private void testSkipListKeySerializer(int delta) throws IOException { 63 | String key = "key-abcdedg" + delta; 64 | String namespace = "namespace-dfsfdafd" + delta; 65 | 66 | byte[] skipListKey = skipListKeySerializer.serialize(key, namespace); 67 | int offset = 10; 68 | byte[] data = new byte[10 + skipListKey.length]; 69 | System.arraycopy(skipListKey, 0, data, offset, skipListKey.length); 70 | MemorySegment skipListKeySegment = MemorySegmentFactory.wrap(data); 71 | assertEquals(key, skipListKeySerializer.deserializeKey(skipListKeySegment, offset, skipListKey.length)); 72 | assertEquals(namespace, skipListKeySerializer.deserializeNamespace(skipListKeySegment, offset, skipListKey.length)); 73 | 74 | Tuple2 serializedKeyAndNamespace = 75 | skipListKeySerializer.getSerializedKeyAndNamespace(skipListKeySegment, offset); 76 | assertEquals(key, deserialize(keySerializer, serializedKeyAndNamespace.f0)); 77 | assertEquals(namespace, deserialize(namespaceSerializer, serializedKeyAndNamespace.f1)); 78 | 79 | byte[] serializedNamespace = skipListKeySerializer.serializeNamespace(namespace); 80 | assertEquals(namespace, deserialize(namespaceSerializer, serializedNamespace)); 81 | } 82 | 83 | @Test 84 | public void testSkipListValueSerializerBasicOp() throws IOException { 85 | testSkipListValueSerializer(0); 86 | } 87 | 88 | @Test 89 | public void testSkipListValueSerializerStateless() throws IOException { 90 | for (int i = 0; i < 10; i++) { 91 | testSkipListValueSerializer(i); 92 | } 93 | } 94 | 95 | private void testSkipListValueSerializer(int i) throws IOException { 96 | String state = "value-" + i; 97 | byte[] value = skipListValueSerializer.serialize(state); 98 | int offset = 10; 99 | byte[] data = new byte[10 + value.length]; 100 | System.arraycopy(value, 0, data, offset, value.length); 101 | assertEquals(state, deserialize(stateSerializer, value)); 102 | assertEquals(state, skipListValueSerializer.deserializeState(MemorySegmentFactory.wrap(data), offset, value.length)); 103 | } 104 | 105 | private T deserialize(TypeSerializer serializer, byte[] data) throws IOException { 106 | ByteArrayInputStreamWithPos inputStream = new ByteArrayInputStreamWithPos(data); 107 | DataInputViewStreamWrapper inputView = new DataInputViewStreamWrapper(inputStream); 108 | return serializer.deserialize(inputView); 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/space/Bucket.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap.space; 20 | 21 | import org.apache.flink.util.MathUtils; 22 | import org.apache.flink.util.Preconditions; 23 | 24 | /** 25 | * A bucket is a continuous space that is divided into blocks with fixed size. 26 | * Buckets can be reused to allocate another size of blocks after it's freed. 27 | */ 28 | public final class Bucket { 29 | 30 | /** 31 | * Base offset of this bucket in the chunk. 32 | */ 33 | private final int baseOffset; 34 | 35 | /** 36 | * Size of this bucket. 37 | */ 38 | private final int bucketSize; 39 | 40 | /** 41 | * Index of block allocator which is using this bucket. -1 indicates 42 | * this bucket is not used by any allocator. 43 | */ 44 | private int blockAllocatorIndex; 45 | 46 | /** 47 | * Size of block to allocate. This field is only meaningful when the 48 | * bucket is used by some allocator. 49 | */ 50 | private int blockSize; 51 | 52 | /** 53 | * Number of bits to represent the block size. 54 | */ 55 | private int blockSizeBits; 56 | 57 | /** 58 | * Number of free blocks in bucket. 59 | */ 60 | private int freeBlockNum; 61 | 62 | /** 63 | * Number of used blocks in bucket. 64 | */ 65 | private int usedBlockNum; 66 | 67 | /** 68 | * An array of indexes for free blocks. Blocks in [0, getFreeBlockNum) 69 | * are free to use. This works like a stack. For allocate, last block 70 | * is popped for the back of array, and for free, block are pushed to 71 | * the back of array. 72 | */ 73 | private int[] freeBlockIndexes; 74 | 75 | Bucket(int offset, int bucketSize) { 76 | this.baseOffset = offset; 77 | this.bucketSize = bucketSize; 78 | this.blockAllocatorIndex = -1; 79 | } 80 | 81 | /** 82 | * Reuse the bucket by the given allocator. 83 | * 84 | * @param bucketAllocatorIndex index of bucket allocator. 85 | */ 86 | void reuseBucket(int bucketAllocatorIndex) { 87 | this.blockAllocatorIndex = bucketAllocatorIndex; 88 | this.blockSize = PowerTwoBucketAllocator.getBlockSizeFromBlockAllocatorIndex(this.blockAllocatorIndex); 89 | Preconditions.checkArgument(this.blockSize <= this.bucketSize, 90 | "Block size can't be larger than bucket size."); 91 | this.blockSizeBits = MathUtils.log2strict(this.blockSize); 92 | 93 | int totalBlockNum = this.bucketSize >>> this.blockSizeBits; 94 | this.freeBlockNum = totalBlockNum; 95 | this.usedBlockNum = 0; 96 | this.freeBlockIndexes = new int[totalBlockNum]; 97 | for (int i = 0; i < totalBlockNum; ++i) { 98 | this.freeBlockIndexes[i] = i; 99 | } 100 | } 101 | 102 | /** 103 | * Allocates a block. 104 | * 105 | * @return offset of the block. 106 | */ 107 | public int allocate() { 108 | // pop the last block 109 | ++usedBlockNum; 110 | --freeBlockNum; 111 | return baseOffset + (freeBlockIndexes[freeBlockNum] << blockSizeBits); 112 | } 113 | 114 | /** 115 | * Frees the block with the given offset. 116 | * 117 | * @param offset offset of the block to free. 118 | */ 119 | public void free(int offset) { 120 | int bucketOffset = offset - baseOffset; 121 | int blockIndex = bucketOffset >>> this.blockSizeBits; 122 | --usedBlockNum; 123 | // push the block to the back 124 | freeBlockIndexes[freeBlockNum++] = blockIndex; 125 | } 126 | 127 | /** 128 | * Returns true if this bucket has free blocks to allocate. 129 | */ 130 | boolean hasFreeBlocks() { 131 | return freeBlockNum > 0; 132 | } 133 | 134 | /** 135 | * Returns true if there is no blocks used in this bucket. 136 | */ 137 | public boolean isFree() { 138 | return usedBlockNum == 0; 139 | } 140 | 141 | int getBlockAllocatorIndex() { 142 | return blockAllocatorIndex; 143 | } 144 | 145 | int getBlockSize() { 146 | return blockSize; 147 | } 148 | 149 | boolean isUninitiated() { 150 | return blockAllocatorIndex == -1; 151 | } 152 | 153 | int getFreeBlockNum() { 154 | return freeBlockNum; 155 | } 156 | 157 | int getUsedBlockNum() { 158 | return usedBlockNum; 159 | } 160 | 161 | int getFreeBytes() { 162 | return freeBlockNum << blockSizeBits; 163 | } 164 | 165 | int getUsedBytes() { 166 | return usedBlockNum << blockSizeBits; 167 | } 168 | 169 | long getBaseOffset() { 170 | return baseOffset; 171 | } 172 | } 173 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/SpillableFoldingState.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap; 20 | 21 | import org.apache.flink.api.common.functions.FoldFunction; 22 | import org.apache.flink.api.common.state.FoldingState; 23 | import org.apache.flink.api.common.state.FoldingStateDescriptor; 24 | import org.apache.flink.api.common.state.State; 25 | import org.apache.flink.api.common.state.StateDescriptor; 26 | import org.apache.flink.api.common.typeutils.TypeSerializer; 27 | import org.apache.flink.runtime.state.StateTransformationFunction; 28 | import org.apache.flink.runtime.state.internal.InternalFoldingState; 29 | import org.apache.flink.util.Preconditions; 30 | 31 | import java.io.IOException; 32 | 33 | /** 34 | * Heap-backed partitioned {@link FoldingState} that is snapshotted into files. 35 | * 36 | * @param The type of the key. 37 | * @param The type of the namespace. 38 | * @param The type of the values that can be folded into the state. 39 | * @param The type of the value in the folding state. 40 | * 41 | * @deprecated will be removed in a future version 42 | */ 43 | @Deprecated 44 | class SpillableFoldingState 45 | extends AbstractHeapAppendingState 46 | implements InternalFoldingState { 47 | /** The function used to fold the state. */ 48 | private final FoldTransformation foldTransformation; 49 | 50 | /** 51 | * Creates a new key/value state for the given hash map of key/value pairs. 52 | * 53 | * @param stateTable The state table for which this state is associated to. 54 | * @param keySerializer The serializer for the keys. 55 | * @param valueSerializer The serializer for the state. 56 | * @param namespaceSerializer The serializer for the namespace. 57 | * @param defaultValue The default value for the state. 58 | * @param foldFunction The fold function used for folding state. 59 | */ 60 | private SpillableFoldingState( 61 | StateTable stateTable, 62 | TypeSerializer keySerializer, 63 | TypeSerializer valueSerializer, 64 | TypeSerializer namespaceSerializer, 65 | ACC defaultValue, 66 | FoldFunction foldFunction) { 67 | super(stateTable, keySerializer, valueSerializer, namespaceSerializer, defaultValue); 68 | this.foldTransformation = new FoldTransformation(foldFunction); 69 | } 70 | 71 | @Override 72 | public TypeSerializer getKeySerializer() { 73 | return keySerializer; 74 | } 75 | 76 | @Override 77 | public TypeSerializer getNamespaceSerializer() { 78 | return namespaceSerializer; 79 | } 80 | 81 | @Override 82 | public TypeSerializer getValueSerializer() { 83 | return valueSerializer; 84 | } 85 | 86 | // ------------------------------------------------------------------------ 87 | // state access 88 | // ------------------------------------------------------------------------ 89 | 90 | @Override 91 | public ACC get() { 92 | return getInternal(); 93 | } 94 | 95 | @Override 96 | public void add(T value) throws IOException { 97 | 98 | if (value == null) { 99 | clear(); 100 | return; 101 | } 102 | 103 | try { 104 | stateTable.transform(currentNamespace, value, foldTransformation); 105 | } catch (Exception e) { 106 | throw new IOException("Could not add value to folding state.", e); 107 | } 108 | } 109 | 110 | private final class FoldTransformation implements StateTransformationFunction { 111 | 112 | private final FoldFunction foldFunction; 113 | 114 | FoldTransformation(FoldFunction foldFunction) { 115 | this.foldFunction = Preconditions.checkNotNull(foldFunction); 116 | } 117 | 118 | @Override 119 | public ACC apply(ACC previousState, T value) throws Exception { 120 | return foldFunction.fold((previousState != null) ? previousState : getDefaultValue(), value); 121 | } 122 | } 123 | 124 | @SuppressWarnings("unchecked") 125 | static IS create( 126 | StateDescriptor stateDesc, 127 | StateTable stateTable, 128 | TypeSerializer keySerializer) { 129 | return (IS) new SpillableFoldingState<>( 130 | stateTable, 131 | keySerializer, 132 | stateTable.getStateSerializer(), 133 | stateTable.getNamespaceSerializer(), 134 | stateDesc.getDefaultValue(), 135 | ((FoldingStateDescriptor) stateDesc).getFoldFunction()); 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/test/java/org/apache/flink/runtime/state/heap/SpillableStateBackendTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * * Licensed to the Apache Software Foundation (ASF) under one 4 | * * or more contributor license agreements. See the NOTICE file 5 | * * distributed with this work for additional information 6 | * * regarding copyright ownership. The ASF licenses this file 7 | * * to you under the Apache License, Version 2.0 (the 8 | * * "License"); you may not use this file except in compliance 9 | * * with the License. You may obtain a copy of the License at 10 | * * 11 | * * http://www.apache.org/licenses/LICENSE-2.0 12 | * * 13 | * * Unless required by applicable law or agreed to in writing, software 14 | * * distributed under the License is distributed on an "AS IS" BASIS, 15 | * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | * * See the License for the specific language governing permissions and 17 | * * limitations under the License. 18 | * 19 | */ 20 | 21 | package org.apache.flink.runtime.state.heap; 22 | 23 | import org.apache.flink.api.common.state.ValueState; 24 | import org.apache.flink.api.common.state.ValueStateDescriptor; 25 | import org.apache.flink.api.common.typeutils.base.IntSerializer; 26 | import org.apache.flink.runtime.checkpoint.CheckpointOptions; 27 | import org.apache.flink.runtime.state.CheckpointStreamFactory; 28 | import org.apache.flink.runtime.state.KeyedStateHandle; 29 | import org.apache.flink.runtime.state.SharedStateRegistry; 30 | import org.apache.flink.runtime.state.SnapshotResult; 31 | import org.apache.flink.runtime.state.StateBackendTestBase; 32 | import org.apache.flink.runtime.state.VoidNamespace; 33 | import org.apache.flink.runtime.state.VoidNamespaceSerializer; 34 | import org.apache.flink.runtime.state.memory.MemoryStateBackend; 35 | 36 | import org.junit.Ignore; 37 | import org.junit.Rule; 38 | import org.junit.Test; 39 | import org.junit.rules.TemporaryFolder; 40 | 41 | import java.io.IOException; 42 | import java.util.concurrent.RunnableFuture; 43 | 44 | import static org.junit.Assert.assertEquals; 45 | import static org.junit.Assert.assertSame; 46 | import static org.junit.Assert.assertTrue; 47 | 48 | /** 49 | * Tests for {@link SpillableStateBackend}. 50 | */ 51 | public class SpillableStateBackendTest extends StateBackendTestBase { 52 | 53 | @Rule 54 | public final TemporaryFolder tempFolder = new TemporaryFolder(); 55 | 56 | @Override 57 | protected SpillableStateBackend getStateBackend() throws IOException { 58 | SpillableStateBackend backend = new SpillableStateBackend(new MemoryStateBackend(true)); 59 | 60 | String dbPath = tempFolder.newFolder().getAbsolutePath(); 61 | backend.setDbStoragePaths(dbPath); 62 | 63 | return backend; 64 | } 65 | 66 | @Override 67 | protected boolean isSerializerPresenceRequiredOnRestore() { 68 | return true; 69 | } 70 | 71 | // disable these because the verification does not work for this state backend 72 | @Override 73 | @Test 74 | public void testValueStateRestoreWithWrongSerializers() {} 75 | 76 | @Override 77 | @Test 78 | public void testListStateRestoreWithWrongSerializers() {} 79 | 80 | @Override 81 | @Test 82 | public void testReducingStateRestoreWithWrongSerializers() {} 83 | 84 | @Override 85 | @Test 86 | public void testMapStateRestoreWithWrongSerializers() {} 87 | 88 | @Ignore 89 | @Test 90 | public void testConcurrentMapIfQueryable() throws Exception { 91 | super.testConcurrentMapIfQueryable(); 92 | } 93 | 94 | @Test 95 | public void testCheckpointManagerWithSnapshotCancellation() throws Exception { 96 | testCheckpointManager(true); 97 | } 98 | 99 | @Test 100 | public void testCheckpointManagerWithNormalSnapshot() throws Exception { 101 | testCheckpointManager(false); 102 | } 103 | 104 | private void testCheckpointManager(boolean cancel) throws Exception { 105 | CheckpointStreamFactory streamFactory = createStreamFactory(); 106 | SharedStateRegistry sharedStateRegistry = new SharedStateRegistry(); 107 | 108 | SpillableKeyedStateBackend backend = 109 | (SpillableKeyedStateBackend) createKeyedBackend(IntSerializer.INSTANCE); 110 | CheckpointManagerImpl checkpointManager = (CheckpointManagerImpl) backend.getCheckpointManager(); 111 | ValueState state = backend.getPartitionedState( 112 | VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, new ValueStateDescriptor<>("id", String.class)); 113 | 114 | // some modifications to the state 115 | backend.setCurrentKey(1); 116 | state.update("1"); 117 | backend.setCurrentKey(2); 118 | state.update("2"); 119 | 120 | RunnableFuture> snapshot = 121 | backend.snapshot(0L, 0L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()); 122 | 123 | assertEquals(1, checkpointManager.getRunningCheckpoints().size()); 124 | RunnableFuture> future = checkpointManager.getRunningCheckpoints().get(0L); 125 | assertSame(snapshot, future); 126 | 127 | if (cancel) { 128 | snapshot.cancel(true); 129 | } else { 130 | runSnapshot(snapshot, sharedStateRegistry); 131 | } 132 | 133 | assertTrue(checkpointManager.getRunningCheckpoints().isEmpty()); 134 | 135 | backend.dispose(); 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/SpillableReducingState.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap; 20 | 21 | import org.apache.flink.api.common.functions.ReduceFunction; 22 | import org.apache.flink.api.common.state.ReducingState; 23 | import org.apache.flink.api.common.state.ReducingStateDescriptor; 24 | import org.apache.flink.api.common.state.State; 25 | import org.apache.flink.api.common.state.StateDescriptor; 26 | import org.apache.flink.api.common.typeutils.TypeSerializer; 27 | import org.apache.flink.runtime.state.StateTransformationFunction; 28 | import org.apache.flink.runtime.state.internal.InternalReducingState; 29 | import org.apache.flink.util.Preconditions; 30 | 31 | import java.io.IOException; 32 | 33 | /** 34 | * Heap-backed partitioned {@link ReducingState} that is snapshotted into files. 35 | * 36 | * @param The type of the key. 37 | * @param The type of the namespace. 38 | * @param The type of the value. 39 | */ 40 | class SpillableReducingState 41 | extends AbstractHeapMergingState 42 | implements InternalReducingState { 43 | 44 | private final ReduceTransformation reduceTransformation; 45 | 46 | /** 47 | * Creates a new key/value state for the given hash map of key/value pairs. 48 | * 49 | * @param stateTable The state table for which this state is associated to. 50 | * @param keySerializer The serializer for the keys. 51 | * @param valueSerializer The serializer for the state. 52 | * @param namespaceSerializer The serializer for the namespace. 53 | * @param defaultValue The default value for the state. 54 | * @param reduceFunction The reduce function used for reducing state. 55 | */ 56 | private SpillableReducingState( 57 | StateTable stateTable, 58 | TypeSerializer keySerializer, 59 | TypeSerializer valueSerializer, 60 | TypeSerializer namespaceSerializer, 61 | V defaultValue, 62 | ReduceFunction reduceFunction) { 63 | 64 | super(stateTable, keySerializer, valueSerializer, namespaceSerializer, defaultValue); 65 | this.reduceTransformation = new ReduceTransformation<>(reduceFunction); 66 | } 67 | 68 | @Override 69 | public TypeSerializer getKeySerializer() { 70 | return keySerializer; 71 | } 72 | 73 | @Override 74 | public TypeSerializer getNamespaceSerializer() { 75 | return namespaceSerializer; 76 | } 77 | 78 | @Override 79 | public TypeSerializer getValueSerializer() { 80 | return valueSerializer; 81 | } 82 | 83 | // ------------------------------------------------------------------------ 84 | // state access 85 | // ------------------------------------------------------------------------ 86 | 87 | @Override 88 | public V get() { 89 | return getInternal(); 90 | } 91 | 92 | @Override 93 | public void add(V value) throws IOException { 94 | 95 | if (value == null) { 96 | clear(); 97 | return; 98 | } 99 | 100 | try { 101 | stateTable.transform(currentNamespace, value, reduceTransformation); 102 | } catch (Exception e) { 103 | throw new IOException("Exception while applying ReduceFunction in reducing state", e); 104 | } 105 | } 106 | 107 | // ------------------------------------------------------------------------ 108 | // state merging 109 | // ------------------------------------------------------------------------ 110 | 111 | @Override 112 | protected V mergeState(V a, V b) throws Exception { 113 | return reduceTransformation.apply(a, b); 114 | } 115 | 116 | static final class ReduceTransformation implements StateTransformationFunction { 117 | 118 | private final ReduceFunction reduceFunction; 119 | 120 | ReduceTransformation(ReduceFunction reduceFunction) { 121 | this.reduceFunction = Preconditions.checkNotNull(reduceFunction); 122 | } 123 | 124 | @Override 125 | public V apply(V previousState, V value) throws Exception { 126 | return previousState != null ? reduceFunction.reduce(previousState, value) : value; 127 | } 128 | } 129 | 130 | @SuppressWarnings("unchecked") 131 | static IS create( 132 | StateDescriptor stateDesc, 133 | StateTable stateTable, 134 | TypeSerializer keySerializer) { 135 | return (IS) new SpillableReducingState<>( 136 | stateTable, 137 | keySerializer, 138 | stateTable.getStateSerializer(), 139 | stateTable.getNamespaceSerializer(), 140 | stateDesc.getDefaultValue(), 141 | ((ReducingStateDescriptor) stateDesc).getReduceFunction()); 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/space/DirectBucketAllocator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap.space; 20 | 21 | import org.apache.flink.annotation.VisibleForTesting; 22 | import org.apache.flink.util.Preconditions; 23 | 24 | import java.util.HashMap; 25 | import java.util.Iterator; 26 | import java.util.Map; 27 | import java.util.Optional; 28 | import java.util.SortedMap; 29 | import java.util.TreeMap; 30 | 31 | import static org.apache.flink.runtime.state.heap.space.SpaceConstants.NO_SPACE; 32 | 33 | /** 34 | * This allocator is aimed to allocate large space, and reduce fragments. 35 | */ 36 | public class DirectBucketAllocator implements BucketAllocator { 37 | 38 | /** 39 | * Capacity of this allocator. 40 | */ 41 | private final int capacity; 42 | 43 | /** 44 | * Map for used spaces. It maps space offset to space size. 45 | */ 46 | private final SortedMap usedSpaces = new TreeMap<>(); 47 | 48 | /** 49 | * Map for free spaces. It maps space offset to space size. 50 | */ 51 | private final SortedMap freeSpaces = new TreeMap<>(); 52 | 53 | /** 54 | * Offset of space that has never been allocated. Size of this 55 | * space is (capacity - offset). 56 | */ 57 | private int offset; 58 | 59 | DirectBucketAllocator(int capacity) { 60 | this.capacity = capacity; 61 | this.offset = 0; 62 | } 63 | 64 | @Override 65 | public synchronized int allocate(int len) { 66 | // 1. try to find a completely matched space 67 | int curOffset = findMatchSpace(len); 68 | if (curOffset != NO_SPACE) { 69 | return curOffset; 70 | } 71 | 72 | // 2. if there is no space that has never been allocated, 73 | // try to compact and find a free space 74 | if (this.offset + len > capacity) { 75 | return compactionAndFindFreeSpace(len); 76 | } 77 | 78 | // 3. allocate space directly 79 | curOffset = this.offset; 80 | this.offset += len; 81 | this.usedSpaces.put(curOffset, len); 82 | return curOffset; 83 | } 84 | 85 | @Override 86 | public synchronized void free(int offset) { 87 | Integer len = usedSpaces.remove(offset); 88 | Preconditions.checkNotNull(len, "Try to free an unused space"); 89 | freeSpaces.put(offset, len); 90 | } 91 | 92 | private int findMatchSpace(int len) { 93 | Optional> result = 94 | this.freeSpaces.entrySet().stream().filter(i -> i.getValue() == len).findFirst(); 95 | if (result.isPresent()) { 96 | int interChunkOffset = result.get().getKey(); 97 | this.usedSpaces.put(interChunkOffset, this.freeSpaces.remove(interChunkOffset)); 98 | return interChunkOffset; 99 | } 100 | return NO_SPACE; 101 | } 102 | 103 | private int compactionAndFindFreeSpace(int len) { 104 | // 1. compact space fragments 105 | Integer lastOffset = -1; 106 | Integer lastLen = -1; 107 | Map needMerge = new HashMap<>(); 108 | Iterator> iterator = freeSpaces.entrySet().iterator(); 109 | while (iterator.hasNext()) { 110 | Map.Entry freeEntry = iterator.next(); 111 | if (lastOffset != -1 && lastOffset + lastLen == freeEntry.getKey()) { 112 | // find offset that can be merged 113 | lastLen = lastLen + freeEntry.getValue(); 114 | needMerge.put(lastOffset, lastLen); 115 | iterator.remove(); 116 | continue; 117 | } 118 | 119 | lastOffset = freeEntry.getKey(); 120 | lastLen = freeEntry.getValue(); 121 | } 122 | needMerge.forEach(this.freeSpaces::put); 123 | 124 | // 2. find max space to allocate 125 | Optional> result = 126 | freeSpaces.entrySet().stream().reduce((x, y) -> x.getValue() > y.getValue() ? x : y); 127 | if (result.isPresent()) { 128 | int interChunkOffset = result.get().getKey(); 129 | int interChunkLen = result.get().getValue(); 130 | if (interChunkLen > len) { 131 | freeSpaces.remove(interChunkOffset); 132 | usedSpaces.put(interChunkOffset, len); 133 | freeSpaces.put(interChunkOffset + len, interChunkLen - len); 134 | return interChunkOffset; 135 | } 136 | } 137 | 138 | return NO_SPACE; 139 | } 140 | 141 | @VisibleForTesting 142 | int getCapacity() { 143 | return capacity; 144 | } 145 | 146 | @VisibleForTesting 147 | int getOffset() { 148 | return offset; 149 | } 150 | 151 | @VisibleForTesting 152 | SortedMap getUsedSpace() { 153 | return usedSpaces; 154 | } 155 | 156 | @VisibleForTesting 157 | SortedMap getFreeSpaces() { 158 | return freeSpaces; 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/test/java/org/apache/flink/runtime/state/heap/TestAllocator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * * Licensed to the Apache Software Foundation (ASF) under one 4 | * * or more contributor license agreements. See the NOTICE file 5 | * * distributed with this work for additional information 6 | * * regarding copyright ownership. The ASF licenses this file 7 | * * to you under the Apache License, Version 2.0 (the 8 | * * "License"); you may not use this file except in compliance 9 | * * with the License. You may obtain a copy of the License at 10 | * * 11 | * * http://www.apache.org/licenses/LICENSE-2.0 12 | * * 13 | * * Unless required by applicable law or agreed to in writing, software 14 | * * distributed under the License is distributed on an "AS IS" BASIS, 15 | * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | * * See the License for the specific language governing permissions and 17 | * * limitations under the License. 18 | * 19 | */ 20 | 21 | package org.apache.flink.runtime.state.heap; 22 | 23 | import org.apache.flink.core.memory.MemorySegment; 24 | import org.apache.flink.core.memory.MemorySegmentFactory; 25 | import org.apache.flink.runtime.state.heap.space.Allocator; 26 | import org.apache.flink.runtime.state.heap.space.Chunk; 27 | import org.apache.flink.runtime.state.heap.space.SpaceUtils; 28 | import org.apache.flink.util.Preconditions; 29 | import org.apache.flink.util.TestLogger; 30 | 31 | import java.util.HashMap; 32 | import java.util.Map; 33 | 34 | import static org.apache.flink.runtime.state.heap.space.Constants.FOUR_BYTES_BITS; 35 | import static org.apache.flink.runtime.state.heap.space.Constants.FOUR_BYTES_MARK; 36 | 37 | /** 38 | * Implementation of {@link Allocator} used for test. This allocator 39 | * will create a chunk for each allocation request. Size of a chunk 40 | * is fixed, and only used by one space. 41 | */ 42 | public class TestAllocator extends TestLogger implements Allocator { 43 | 44 | /** 45 | * Max allocate size supported by this allocator. 46 | */ 47 | private final int maxAllocateSize; 48 | 49 | private Map chunkMap; 50 | 51 | private volatile int chunkCounter; 52 | 53 | public TestAllocator() { 54 | this(256); 55 | } 56 | 57 | public TestAllocator(int maxAllocateSize) { 58 | this.maxAllocateSize = maxAllocateSize; 59 | this.chunkMap = new HashMap<>(); 60 | this.chunkCounter = 0; 61 | } 62 | 63 | @Override 64 | public synchronized long allocate(int size) { 65 | Preconditions.checkArgument(size <= maxAllocateSize, 66 | "Can't allocate size of " + size + " larger than maxAllocateSize " + maxAllocateSize); 67 | int chunkId = chunkCounter++; 68 | TestChunk testChunk = new TestChunk(chunkId, maxAllocateSize); 69 | chunkMap.put(chunkId, testChunk); 70 | int offset = testChunk.allocate(size); 71 | return ((chunkId & FOUR_BYTES_MARK) << FOUR_BYTES_BITS) | (offset & FOUR_BYTES_MARK); 72 | } 73 | 74 | @Override 75 | public synchronized void free(long address) { 76 | int chunkId = SpaceUtils.getChunkIdByAddress(address); 77 | int offset = SpaceUtils.getChunkOffsetByAddress(address); 78 | TestChunk chunk = chunkMap.remove(chunkId); 79 | if (chunk != null) { 80 | chunk.free(offset); 81 | } 82 | } 83 | 84 | @Override 85 | public synchronized Chunk getChunkById(int chunkId) { 86 | TestChunk chunk = chunkMap.get(chunkId); 87 | Preconditions.checkNotNull(chunk, "chunk " + chunkId + " doest not exist."); 88 | return chunk; 89 | } 90 | 91 | @Override 92 | public synchronized void close() { 93 | chunkMap.clear(); 94 | } 95 | 96 | public int getMaxAllocateSize() { 97 | return maxAllocateSize; 98 | } 99 | 100 | /** 101 | * Returns total size of used space. 102 | */ 103 | public synchronized int getTotalSpaceSize() { 104 | return chunkMap.size() * maxAllocateSize; 105 | } 106 | 107 | /** 108 | * Returns number of used space. 109 | */ 110 | public synchronized int getTotalSpaceNumber() { 111 | return chunkMap.size(); 112 | } 113 | 114 | /** 115 | * Implementation of {@link Chunk} used for test. A chunk can only be used 116 | * by one space. 117 | */ 118 | public static class TestChunk implements Chunk { 119 | 120 | private final int chunkId; 121 | private final int size; 122 | private final MemorySegment memorySegment; 123 | private final int offset; 124 | private volatile boolean used; 125 | 126 | TestChunk(int chunkId, int size) { 127 | this.offset = 14; 128 | this.chunkId = chunkId; 129 | this.size = size + offset; 130 | this.memorySegment = MemorySegmentFactory.allocateUnpooledSegment(size); 131 | } 132 | 133 | @Override 134 | public synchronized int allocate(int len) { 135 | Preconditions.checkState(!used, "chunk has been allocated."); 136 | Preconditions.checkState(len <= size, "There is no enough size."); 137 | used = true; 138 | return offset; 139 | } 140 | 141 | @Override 142 | public synchronized void free(int interChunkOffset) { 143 | used = false; 144 | } 145 | 146 | @Override 147 | public int getChunkId() { 148 | return chunkId; 149 | } 150 | 151 | @Override 152 | public int getChunkCapacity() { 153 | return size; 154 | } 155 | 156 | @Override 157 | public MemorySegment getMemorySegment(int chunkOffset) { 158 | return memorySegment; 159 | } 160 | 161 | @Override 162 | public int getOffsetInSegment(int offsetInChunk) { 163 | return offsetInChunk; 164 | } 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/SpillableAggregatingState.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap; 20 | 21 | import org.apache.flink.api.common.functions.AggregateFunction; 22 | import org.apache.flink.api.common.state.AggregatingState; 23 | import org.apache.flink.api.common.state.AggregatingStateDescriptor; 24 | import org.apache.flink.api.common.state.State; 25 | import org.apache.flink.api.common.state.StateDescriptor; 26 | import org.apache.flink.api.common.typeutils.TypeSerializer; 27 | import org.apache.flink.runtime.state.StateTransformationFunction; 28 | import org.apache.flink.runtime.state.internal.InternalAggregatingState; 29 | import org.apache.flink.util.Preconditions; 30 | 31 | import java.io.IOException; 32 | 33 | /** 34 | * Heap-backed partitioned {@link AggregatingState} that is 35 | * snapshotted into files. 36 | * 37 | * @param The type of the key. 38 | * @param The type of the namespace. 39 | * @param The type of the value added to the state. 40 | * @param The type of the value stored in the state (the accumulator type). 41 | * @param The type of the value returned from the state. 42 | */ 43 | class SpillableAggregatingState 44 | extends AbstractHeapMergingState 45 | implements InternalAggregatingState { 46 | private final AggregateTransformation aggregateTransformation; 47 | 48 | /** 49 | * Creates a new key/value state for the given hash map of key/value pairs. 50 | * 51 | * @param stateTable The state table for which this state is associated to. 52 | * @param keySerializer The serializer for the keys. 53 | * @param valueSerializer The serializer for the state. 54 | * @param namespaceSerializer The serializer for the namespace. 55 | * @param defaultValue The default value for the state. 56 | * @param aggregateFunction The aggregating function used for aggregating state. 57 | */ 58 | private SpillableAggregatingState( 59 | StateTable stateTable, 60 | TypeSerializer keySerializer, 61 | TypeSerializer valueSerializer, 62 | TypeSerializer namespaceSerializer, 63 | ACC defaultValue, 64 | AggregateFunction aggregateFunction) { 65 | 66 | super(stateTable, keySerializer, valueSerializer, namespaceSerializer, defaultValue); 67 | this.aggregateTransformation = new AggregateTransformation<>(aggregateFunction); 68 | } 69 | 70 | @Override 71 | public TypeSerializer getKeySerializer() { 72 | return keySerializer; 73 | } 74 | 75 | @Override 76 | public TypeSerializer getNamespaceSerializer() { 77 | return namespaceSerializer; 78 | } 79 | 80 | @Override 81 | public TypeSerializer getValueSerializer() { 82 | return valueSerializer; 83 | } 84 | 85 | // ------------------------------------------------------------------------ 86 | // state access 87 | // ------------------------------------------------------------------------ 88 | 89 | @Override 90 | public OUT get() { 91 | ACC accumulator = getInternal(); 92 | return accumulator != null ? aggregateTransformation.aggFunction.getResult(accumulator) : null; 93 | } 94 | 95 | @Override 96 | public void add(IN value) throws IOException { 97 | final N namespace = currentNamespace; 98 | 99 | if (value == null) { 100 | clear(); 101 | return; 102 | } 103 | 104 | try { 105 | stateTable.transform(namespace, value, aggregateTransformation); 106 | } catch (Exception e) { 107 | throw new IOException("Exception while applying AggregateFunction in aggregating state", e); 108 | } 109 | } 110 | 111 | // ------------------------------------------------------------------------ 112 | // state merging 113 | // ------------------------------------------------------------------------ 114 | 115 | @Override 116 | protected ACC mergeState(ACC a, ACC b) { 117 | return aggregateTransformation.aggFunction.merge(a, b); 118 | } 119 | 120 | static final class AggregateTransformation implements StateTransformationFunction { 121 | 122 | private final AggregateFunction aggFunction; 123 | 124 | AggregateTransformation(AggregateFunction aggFunction) { 125 | this.aggFunction = Preconditions.checkNotNull(aggFunction); 126 | } 127 | 128 | @Override 129 | public ACC apply(ACC accumulator, IN value) { 130 | if (accumulator == null) { 131 | accumulator = aggFunction.createAccumulator(); 132 | } 133 | return aggFunction.add(value, accumulator); 134 | } 135 | } 136 | 137 | @SuppressWarnings("unchecked") 138 | static IS create( 139 | StateDescriptor stateDesc, 140 | StateTable stateTable, 141 | TypeSerializer keySerializer) { 142 | return (IS) new SpillableAggregatingState<>( 143 | stateTable, 144 | keySerializer, 145 | stateTable.getStateSerializer(), 146 | stateTable.getNamespaceSerializer(), 147 | stateDesc.getDefaultValue(), 148 | ((AggregatingStateDescriptor) stateDesc).getAggregateFunction()); 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/test/java/org/apache/flink/runtime/state/heap/OnHeapLevelIndexHeaderTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * * Licensed to the Apache Software Foundation (ASF) under one 4 | * * or more contributor license agreements. See the NOTICE file 5 | * * distributed with this work for additional information 6 | * * regarding copyright ownership. The ASF licenses this file 7 | * * to you under the Apache License, Version 2.0 (the 8 | * * "License"); you may not use this file except in compliance 9 | * * with the License. You may obtain a copy of the License at 10 | * * 11 | * * http://www.apache.org/licenses/LICENSE-2.0 12 | * * 13 | * * Unless required by applicable law or agreed to in writing, software 14 | * * distributed under the License is distributed on an "AS IS" BASIS, 15 | * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | * * See the License for the specific language governing permissions and 17 | * * limitations under the License. 18 | * 19 | */ 20 | 21 | package org.apache.flink.runtime.state.heap; 22 | 23 | import org.apache.flink.util.TestLogger; 24 | 25 | import org.junit.Assert; 26 | import org.junit.Before; 27 | import org.junit.Test; 28 | 29 | import java.util.concurrent.ThreadLocalRandom; 30 | 31 | import static org.apache.flink.runtime.state.heap.SkipListUtils.DEFAULT_LEVEL; 32 | import static org.apache.flink.runtime.state.heap.SkipListUtils.MAX_LEVEL; 33 | import static org.apache.flink.runtime.state.heap.SkipListUtils.NIL_NODE; 34 | 35 | /** 36 | * Tests for {@link OnHeapLevelIndexHeader}. 37 | */ 38 | public class OnHeapLevelIndexHeaderTest extends TestLogger { 39 | private static final ThreadLocalRandom random = ThreadLocalRandom.current(); 40 | 41 | private OnHeapLevelIndexHeader heapHeadIndex; 42 | 43 | @Before 44 | public void setUp() { 45 | heapHeadIndex = new OnHeapLevelIndexHeader(); 46 | } 47 | 48 | @Test 49 | public void testInitStatus() { 50 | Assert.assertEquals(1, heapHeadIndex.getLevel()); 51 | Assert.assertEquals(DEFAULT_LEVEL, heapHeadIndex.getLevelIndex().length); 52 | for (long node : heapHeadIndex.getLevelIndex()) { 53 | Assert.assertEquals(NIL_NODE, node); 54 | } 55 | for (int level = 0; level <= heapHeadIndex.getLevel(); level++) { 56 | Assert.assertEquals(NIL_NODE, heapHeadIndex.getNextNode(level)); 57 | } 58 | } 59 | 60 | @Test 61 | public void testNormallyUpdateLevel() { 62 | int level = heapHeadIndex.getLevel(); 63 | // update level to no more than init max level 64 | for (; level <= DEFAULT_LEVEL; level++) { 65 | heapHeadIndex.updateLevel(level); 66 | Assert.assertEquals(level, heapHeadIndex.getLevel()); 67 | Assert.assertEquals(DEFAULT_LEVEL, heapHeadIndex.getLevelIndex().length); 68 | } 69 | // update level to trigger scale up 70 | heapHeadIndex.updateLevel(level); 71 | Assert.assertEquals(level, heapHeadIndex.getLevel()); 72 | Assert.assertEquals(DEFAULT_LEVEL * 2, heapHeadIndex.getLevelIndex().length); 73 | } 74 | 75 | /** 76 | * Test update to current level is allowed. 77 | */ 78 | @Test 79 | public void testUpdateToCurrentLevel() { 80 | heapHeadIndex.updateLevel(heapHeadIndex.getLevel()); 81 | } 82 | 83 | /** 84 | * Test update to current level is allowed. 85 | */ 86 | @Test 87 | public void testUpdateLevelToLessThanCurrentLevel() { 88 | int level = heapHeadIndex.getLevel(); 89 | // update level 10 times 90 | for (int i = 0; i < 10; i++) { 91 | heapHeadIndex.updateLevel(++level); 92 | } 93 | // check update level to values less than current top level 94 | for (int i = level - 1; i >= 0; i--) { 95 | heapHeadIndex.updateLevel(i); 96 | Assert.assertEquals(level, heapHeadIndex.getLevel()); 97 | } 98 | } 99 | 100 | /** 101 | * Test once update more than one level is not allowed. 102 | */ 103 | @Test 104 | public void testOnceUpdateMoreThanOneLevel() { 105 | try { 106 | heapHeadIndex.updateLevel(heapHeadIndex.getLevel() + 2); 107 | Assert.fail("Should have thrown exception"); 108 | } catch (Exception e) { 109 | Assert.assertTrue(e instanceof IllegalArgumentException); 110 | } 111 | } 112 | 113 | /** 114 | * Test update to negative level is not allowed. 115 | */ 116 | @Test 117 | public void testUpdateToNegativeLevel() { 118 | try { 119 | heapHeadIndex.updateLevel(-1); 120 | Assert.fail("Should throw exception"); 121 | } catch (Exception e) { 122 | Assert.assertTrue(e instanceof IllegalArgumentException); 123 | } 124 | } 125 | 126 | /** 127 | * Test update to more than max level is not allowed. 128 | */ 129 | @Test 130 | public void testUpdateToMoreThanMaximumAllowed() { 131 | try { 132 | heapHeadIndex.updateLevel(MAX_LEVEL + 1); 133 | Assert.fail("Should throw exception"); 134 | } catch (Exception e) { 135 | Assert.assertTrue(e instanceof IllegalArgumentException); 136 | } 137 | } 138 | 139 | @Test 140 | public void testUpdateNextNode() { 141 | // test update next node of level 0 142 | int level = 0; 143 | long node1 = random.nextLong(Long.MAX_VALUE); 144 | heapHeadIndex.updateNextNode(level, node1); 145 | Assert.assertEquals(node1, heapHeadIndex.getNextNode(level)); 146 | // Increase one level and make sure everything still works 147 | heapHeadIndex.updateLevel(++level); 148 | long node2 = random.nextLong(Long.MAX_VALUE); 149 | heapHeadIndex.updateNextNode(level, node2); 150 | Assert.assertEquals(node2, heapHeadIndex.getNextNode(level)); 151 | Assert.assertEquals(node1, heapHeadIndex.getNextNode(level - 1)); 152 | } 153 | 154 | @Test 155 | public void testUpdateNextNodeAfterScale() { 156 | int level = 0; 157 | for (; level <= DEFAULT_LEVEL; level++) { 158 | heapHeadIndex.updateLevel(level); 159 | } 160 | heapHeadIndex.updateLevel(level); 161 | long node = random.nextLong(Long.MAX_VALUE); 162 | heapHeadIndex.updateNextNode(level, node); 163 | Assert.assertEquals(node, heapHeadIndex.getNextNode(level)); 164 | for (int i = 0; i < level; i++) { 165 | Assert.assertEquals(NIL_NODE, heapHeadIndex.getNextNode(i)); 166 | } 167 | } 168 | 169 | } 170 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/space/PowerTwoBucketAllocator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap.space; 20 | 21 | import org.apache.flink.annotation.VisibleForTesting; 22 | import org.apache.flink.util.MathUtils; 23 | import org.apache.flink.util.Preconditions; 24 | 25 | import java.util.concurrent.ConcurrentLinkedQueue; 26 | 27 | /** 28 | * This is a simple buddy-like allocator used for small space. The allocator always allocates a block 29 | * of power-of-two size to satisfy a space request. For example, if a space of 54 bytes is requested, 30 | * allocator will allocate a block of 64 bytes to it which leads to a waste of 10 bytes. 31 | * 32 | *

A {@link Chunk} will be divided into multiple buckets of the same size. Each bucket is further 33 | * divided into blocks. Blocks in a bucket have the same size, but blocks in different buckets may have 34 | * different size. Size of each block must be a power of two, and the maximum size of block can't exceed 35 | * bucket size. 36 | */ 37 | public class PowerTwoBucketAllocator implements BucketAllocator { 38 | 39 | /** 40 | * Minimum size of a block. 41 | */ 42 | private static final int MIN_BLOCK_SIZE = 32; 43 | 44 | /** 45 | * Number of bits to represent {@link PowerTwoBucketAllocator#MIN_BLOCK_SIZE}. 46 | */ 47 | private static final int MIN_BLOCK_SIZE_BITS = MathUtils.log2strict(MIN_BLOCK_SIZE); 48 | 49 | /** 50 | * Number of bits to represent bucket size. 51 | */ 52 | private final int bucketSizeBits; 53 | 54 | /** 55 | * An array of all buckets. 56 | */ 57 | private Bucket[] allBuckets; 58 | 59 | /** 60 | * An array of block allocator, and each allocator can only allocate blocks with fixed size. 61 | * Size of blocks allocated by a allocator is twice of that allocated by the former allocator 62 | * in the array. For example, size of blocks allocated by allocators in the array can be 32, 63 | * 64, 128. 64 | */ 65 | private BlockAllocator[] blockAllocators; 66 | 67 | /** 68 | * A queue of free buckets. 69 | */ 70 | private final ConcurrentLinkedQueue freeBucketsQueue; 71 | 72 | PowerTwoBucketAllocator(int chunkSize, int bucketSize) { 73 | Preconditions.checkArgument((bucketSize & bucketSize - 1) == 0, 74 | "Bucket size must be a power of 2, but the actual is " + bucketSize); 75 | this.bucketSizeBits = MathUtils.log2strict(bucketSize); 76 | this.freeBucketsQueue = new ConcurrentLinkedQueue<>(); 77 | 78 | int numberBuckets = chunkSize >>> this.bucketSizeBits; 79 | // number of possible values for block size 80 | int numberBlockSize = bucketSizeBits - MIN_BLOCK_SIZE_BITS + 1; 81 | Preconditions.checkArgument(numberBuckets >= numberBlockSize, 82 | "Number of possible values for block size is more than the number of buckets, " + 83 | "so allocator can't satisfy requests for different block size at the same time. " + 84 | "Try to use a larger chunk."); 85 | this.allBuckets = new Bucket[numberBuckets]; 86 | for (int i = 0; i < allBuckets.length; ++i) { 87 | allBuckets[i] = new Bucket(i << this.bucketSizeBits, bucketSize); 88 | this.freeBucketsQueue.offer(allBuckets[i]); 89 | } 90 | 91 | this.blockAllocators = new BlockAllocator[this.bucketSizeBits - MIN_BLOCK_SIZE_BITS + 1]; 92 | for (int i = 0; i < blockAllocators.length; i++) { 93 | blockAllocators[i] = new BlockAllocator(i, this.freeBucketsQueue); 94 | } 95 | } 96 | 97 | @Override 98 | public int allocate(int len) { 99 | Preconditions.checkArgument(len > 0, 100 | "Size to allocate must be positive, but the actual is " + len); 101 | int blockAllocatorIndex = getBlockAllocatorIndex(len); 102 | if (blockAllocatorIndex >= blockAllocators.length) { 103 | throw new RuntimeException("PowerTwoBucketAllocator can't allocate size larger than bucket size"); 104 | } 105 | return blockAllocators[blockAllocatorIndex].allocateBlock(); 106 | } 107 | 108 | @Override 109 | public void free(int offset) { 110 | int bucketIndex = getBucketIndex(offset); 111 | Bucket bucket = allBuckets[bucketIndex]; 112 | blockAllocators[bucket.getBlockAllocatorIndex()].freeBlock(bucket, offset); 113 | } 114 | 115 | @VisibleForTesting 116 | int getBucketIndex(int offset) { 117 | return offset >>> bucketSizeBits; 118 | } 119 | 120 | @VisibleForTesting 121 | Bucket[] getAllBuckets() { 122 | return allBuckets; 123 | } 124 | 125 | @VisibleForTesting 126 | BlockAllocator[] getBlockAllocators() { 127 | return blockAllocators; 128 | } 129 | 130 | @VisibleForTesting 131 | ConcurrentLinkedQueue getFreeBucketsQueue() { 132 | return freeBucketsQueue; 133 | } 134 | 135 | /** 136 | * Returns the index of block allocator responsible for the given block size in 137 | * {@link PowerTwoBucketAllocator#blockAllocators}. 138 | */ 139 | @VisibleForTesting 140 | static int getBlockAllocatorIndex(int blockSize) { 141 | Preconditions.checkArgument(blockSize > 0, "Block size must be positive"); 142 | if (blockSize <= MIN_BLOCK_SIZE) { 143 | return 0; 144 | } 145 | return MathUtils.log2strict(MathUtils.roundUpToPowerOfTwo(blockSize)) - MIN_BLOCK_SIZE_BITS; 146 | } 147 | 148 | /** 149 | * Returns block size of the block allocator with the given index. 150 | */ 151 | @VisibleForTesting 152 | static int getBlockSizeFromBlockAllocatorIndex(int blockAllocatorIndex) { 153 | return 1 << (blockAllocatorIndex + MIN_BLOCK_SIZE_BITS); 154 | } 155 | 156 | } 157 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # A Spillable State Backend for Apache Flink 2 | 3 | ## Introduction 4 | 5 | `HeapKeyedStateBackend` is one of the two `KeyedStateBackend` in Flink, since state lives as Java objects on the heap in `HeapKeyedStateBackend` and the de/serialization only happens during state snapshot and restore, it outperforms `RocksDBKeyeStateBackend` when all data could reside in memory. 6 | 7 | However, along with the advantage, `HeapKeyedStateBackend` also has its shortcomings, and the most painful one is the difficulty to estimate the maximum heap size (Xmx) to set, and we will suffer from GC impact once the heap memory is not enough to hold all state data. There’re several (inevitable) causes for such scenario, including (but not limited to): 8 | 9 | * Memory overhead of Java object representation (tens of times of the serialized data size). 10 | * Data flood caused by burst traffic. 11 | * Data accumulation caused by source malfunction. 12 | 13 | To resolve this problem, we proposed a new `SpillableKeyedStateBackend` to support spilling state data to disk before heap memory is exhausted. We will monitor the heap usage and choose the coldest data to spill, and reload them when heap memory is regained after data removing or TTL expiration, automatically. 14 | 15 | Similar to the idea of [Anti-Caching approach](http://www.vldb.org/pvldb/vol6/p1942-debrabant.pdf) proposed for database, the main difference between supporting data spilling in `HeapKeyedStateBackend` and adding a big cache for `RocksDBKeyedStateBackend` is now memory is the primary storage device than disk. Data is either in memory or on disk instead of in both places at the same time thus saving the cost to prevent inconsistency, and rather than starting with data on disk and reading hot data into cache, data starts in memory and cold data is evicted to disk. 16 | 17 | More details please refer to [FLIP-50](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=125307861), and the upstreaming work is in progress through [FLINK-12692](https://issues.apache.org/jira/browse/FLINK-12692). 18 | We setup this repository as a preview version for those who want to try it out before Apache Flink officially supports it. 19 | 20 | ## How to build the binary 21 | 22 | Please note that we only support Flink 1.10.x and later releases. 23 | 24 | You can use the below commands to build a binary to run with Flink 1.10.x releases: 25 | 26 | ``` 27 | git clone https://github.com/realtime-storage-engine/flink-spillable-statebackend.git 28 | cd flink-spillable-statebackend/flink-statebackend-heap-spillable 29 | git checkout origin/release-1.10 30 | mvn clean package -Dflink.version= 31 | ``` 32 | 33 | And the below commands against current Flink master: 34 | 35 | ``` 36 | git clone https://github.com/realtime-storage-engine/flink-spillable-statebackend.git 37 | cd flink-spillable-statebackend/flink-statebackend-heap-spillable 38 | mvn clean package -Dflink.version= 39 | ``` 40 | 41 | The JAR file will be generated in `target` directory with name of `flink-statebackend-heap-spillable-`. 42 | For example, `flink-statebackend-heap-spillable-1.10.1.jar` for Flink 1.10.1. 43 | 44 | ## How to deploy 45 | 46 | First of all, copy the compiled JAR file into the `lib` directory of your Flink deployment 47 | 48 | And then set your state backend to `SpillableStateBackend`. There are two ways to achieve this: 49 | 50 | 1. Set `state.backend` to `org.apache.flink.runtime.state.heap.SpillableStateBackendFactory` in flink-conf.yaml 51 | 52 | 2. Set via java API 53 | 54 | ```java 55 | StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); 56 | env.setStateBackend(new SpillableStateBackend(checkpointPath)); 57 | ``` 58 | 59 | And you need to add the below dependency to your application before compilation: 60 | 61 | ```xml 62 | 63 | org.apache.flink 64 | flink-statebackend-heap-spillable 65 | ${flink.version} 66 | 67 | ``` 68 | 69 | ## Configurations 70 | 71 | | Key | Default | Type | Description | 72 | | ------------- | ------ | --------- | ---- | 73 | |state.backend.spillable.chunk-size|512MB|MemorySize|Size of a chunk (mmap file) which should be a power of two| 74 | |state.backend.spillable.heap-status.check-interval|1min|Duration|Interval to check heap status| 75 | |state.backend.spillable.gc-time.threshold|2s|Duration|If garbage collection time exceeds the threshold, state will be spilled| 76 | |state.backend.spillable.spill-size.ratio|0.2|Float|Ratio of retained state to spill in a turn| 77 | |state.backend.spillable.load-start.ratio|0.1|Float|If memory usage is under this watermark, state will be loaded into memory | 78 | |state.backend.spillable.load-end.ratio|0.3|Float|Memory usage can't exceed this watermark after state is loaded| 79 | |state.backend.spillable.trigger-interval|1min|Duration|Interval to trigger continuous spill/load| 80 | |state.backend.spillable.cancel.checkpoint|true|Boolean|Whether to cancel running checkpoint before spill. Cancelling checkpoints can release the spilled states and make them gc faster| 81 | 82 | ## Performance 83 | 84 | We use a `WordCount` benchmark to compare performance of `HeapKeyedStateBackend`, `SpillableKeyedStateBackend` and `RocksDBKeyedStateBackend`. 85 | You can find the source code of the benchmark as well as how to execute it in the `flink-spillable-benchmark` module of this project. 86 | 87 | With the below hardware environment and job setup: 88 | 89 | * Machine 90 | * 96 Cores, Intel(R) Xeon(R) Platinum 8163 CPU @ 2.50GHz 91 | * 512GB memory 92 | * 12 x 7.3T HDD storage 93 | * Job 94 | * One slot per TaskManager 95 | * Job parallelism: 1 96 | * Disable slot sharing and operator chaining 97 | * Checkpoint interval: 1min 98 | * 20 Million keys, and each key is a 16-length `String` 99 | * Word sending rate: 1M words/s 100 | 101 | * StateBackend 102 | * `HeapKeyedStateBackend`: 10GB heap 103 | * `SpillableStateBackend`: 3GB heap, with the default configuration 104 | * `RocksDBStateBackend`: 3GB block cache, disable `state.backend.rocksdb.memory.managed`, all other configurations with default setting 105 | 106 | we got the below result: 107 | 108 | | StateBackend | TPS | Description | 109 | | ------------- | ---------- | ---- | 110 | |`HeapKeyedStateBackend`|720K records/s|| 111 | |`SpillableStateBackend`|130K records/s|47 out of 128 (36.71%) key groups are spilled to disk| 112 | |`RocksDBStateBackend`|60K records/s|| 113 | 114 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/test/java/org/apache/flink/runtime/state/heap/SnapshotCompatibilityTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * * Licensed to the Apache Software Foundation (ASF) under one 4 | * * or more contributor license agreements. See the NOTICE file 5 | * * distributed with this work for additional information 6 | * * regarding copyright ownership. The ASF licenses this file 7 | * * to you under the Apache License, Version 2.0 (the 8 | * * "License"); you may not use this file except in compliance 9 | * * with the License. You may obtain a copy of the License at 10 | * * 11 | * * http://www.apache.org/licenses/LICENSE-2.0 12 | * * 13 | * * Unless required by applicable law or agreed to in writing, software 14 | * * distributed under the License is distributed on an "AS IS" BASIS, 15 | * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | * * See the License for the specific language governing permissions and 17 | * * limitations under the License. 18 | * 19 | */ 20 | 21 | package org.apache.flink.runtime.state.heap; 22 | 23 | import org.apache.flink.api.common.state.StateDescriptor; 24 | import org.apache.flink.api.common.typeutils.TypeSerializer; 25 | import org.apache.flink.api.common.typeutils.base.IntSerializer; 26 | import org.apache.flink.configuration.Configuration; 27 | import org.apache.flink.core.memory.ByteArrayInputStreamWithPos; 28 | import org.apache.flink.core.memory.ByteArrayOutputStreamWithPos; 29 | import org.apache.flink.core.memory.DataInputViewStreamWrapper; 30 | import org.apache.flink.core.memory.DataOutputViewStreamWrapper; 31 | import org.apache.flink.runtime.state.ArrayListSerializer; 32 | import org.apache.flink.runtime.state.KeyGroupRange; 33 | import org.apache.flink.runtime.state.KeyedBackendSerializationProxy; 34 | import org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo; 35 | import org.apache.flink.runtime.state.StateEntry; 36 | import org.apache.flink.runtime.state.StateSnapshot; 37 | import org.apache.flink.runtime.state.StateSnapshotKeyGroupReader; 38 | import org.apache.flink.runtime.state.heap.space.SpaceAllocator; 39 | 40 | import org.junit.Assert; 41 | import org.junit.Rule; 42 | import org.junit.Test; 43 | import org.junit.rules.TemporaryFolder; 44 | 45 | import java.io.File; 46 | import java.io.IOException; 47 | import java.util.ArrayList; 48 | import java.util.HashMap; 49 | import java.util.Random; 50 | 51 | /** 52 | * TODO This test is just to verify the snapshot format, and should be removed finllay. 53 | * Because spillable state backend may have it's own snapshot format. 54 | */ 55 | public class SnapshotCompatibilityTest { 56 | 57 | private final TypeSerializer keySerializer = IntSerializer.INSTANCE; 58 | 59 | @Rule 60 | public final TemporaryFolder tmp = new TemporaryFolder(); 61 | 62 | @Test 63 | public void checkCompatibleSerializationFormats() throws IOException { 64 | final Random r = new Random(42); 65 | RegisteredKeyValueStateBackendMetaInfo> metaInfo = 66 | new RegisteredKeyValueStateBackendMetaInfo<>( 67 | StateDescriptor.Type.VALUE, 68 | "test", 69 | IntSerializer.INSTANCE, 70 | new ArrayListSerializer<>(IntSerializer.INSTANCE)); 71 | 72 | final MockInternalKeyContext keyContext = new MockInternalKeyContext<>(); 73 | 74 | CopyOnWriteStateTable> cowStateTable = 75 | new CopyOnWriteStateTable<>(keyContext, metaInfo, keySerializer); 76 | 77 | for (int i = 0; i < 100; ++i) { 78 | ArrayList list = new ArrayList<>(5); 79 | int end = r.nextInt(5); 80 | for (int j = 0; j < end; ++j) { 81 | list.add(r.nextInt(100)); 82 | } 83 | 84 | keyContext.setCurrentKey(r.nextInt(10)); 85 | cowStateTable.put(r.nextInt(2), list); 86 | } 87 | 88 | StateSnapshot snapshot = cowStateTable.stateSnapshot(); 89 | 90 | Configuration configuration = new Configuration(); 91 | configuration.set(SpillableOptions.SPACE_TYPE, SpaceAllocator.SpaceType.MMAP.name()); 92 | SpaceAllocator spaceAllocator = new SpaceAllocator(configuration, new File[] {tmp.newFolder()}); 93 | SpillAndLoadManagerImpl spillAndLoadManager = new SpillAndLoadManagerImpl( 94 | new SpillAndLoadManagerImpl.StateTableContainerImpl<>(new HashMap<>()), 95 | new TestHeapStatusMonitor(Long.MAX_VALUE), new CheckpointManagerImpl(), new Configuration()); 96 | final SpillableStateTableImpl> nestedMapsStateTable = 97 | new SpillableStateTableImpl<>(keyContext, metaInfo, keySerializer, spaceAllocator, spillAndLoadManager); 98 | 99 | restoreStateTableFromSnapshot(nestedMapsStateTable, snapshot, keyContext.getKeyGroupRange()); 100 | snapshot.release(); 101 | 102 | Assert.assertEquals(cowStateTable.size(), nestedMapsStateTable.size()); 103 | for (StateEntry> entry : cowStateTable) { 104 | Assert.assertEquals(entry.getState(), nestedMapsStateTable.get(entry.getKey(), entry.getNamespace())); 105 | } 106 | 107 | snapshot = nestedMapsStateTable.stateSnapshot(); 108 | cowStateTable = new CopyOnWriteStateTable<>(keyContext, metaInfo, keySerializer); 109 | 110 | restoreStateTableFromSnapshot(cowStateTable, snapshot, keyContext.getKeyGroupRange()); 111 | snapshot.release(); 112 | 113 | Assert.assertEquals(nestedMapsStateTable.size(), cowStateTable.size()); 114 | for (StateEntry> entry : cowStateTable) { 115 | Assert.assertEquals(nestedMapsStateTable.get(entry.getKey(), entry.getNamespace()), entry.getState()); 116 | } 117 | } 118 | 119 | private void restoreStateTableFromSnapshot( 120 | StateTable> stateTable, 121 | StateSnapshot snapshot, 122 | KeyGroupRange keyGroupRange) throws IOException { 123 | 124 | final ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos(1024 * 1024); 125 | final DataOutputViewStreamWrapper dov = new DataOutputViewStreamWrapper(out); 126 | final StateSnapshot.StateKeyGroupWriter keyGroupPartitionedSnapshot = snapshot.getKeyGroupWriter(); 127 | for (Integer keyGroup : keyGroupRange) { 128 | keyGroupPartitionedSnapshot.writeStateInKeyGroup(dov, keyGroup); 129 | } 130 | 131 | final ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(out.getBuf()); 132 | final DataInputViewStreamWrapper div = new DataInputViewStreamWrapper(in); 133 | 134 | final StateSnapshotKeyGroupReader keyGroupReader = 135 | StateTableByKeyGroupReaders.readerForVersion(stateTable, KeyedBackendSerializationProxy.VERSION); 136 | 137 | for (Integer keyGroup : keyGroupRange) { 138 | keyGroupReader.readMappingsInKeyGroup(div, keyGroup); 139 | } 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/SpillableOptions.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap; 20 | 21 | import org.apache.flink.configuration.ConfigOption; 22 | import org.apache.flink.configuration.ConfigOptions; 23 | import org.apache.flink.configuration.Configuration; 24 | import org.apache.flink.configuration.MemorySize; 25 | import org.apache.flink.configuration.ReadableConfig; 26 | import org.apache.flink.runtime.state.heap.space.SpaceAllocator; 27 | 28 | import java.time.Duration; 29 | 30 | /** 31 | * Options for space allocation. 32 | */ 33 | public class SpillableOptions { 34 | 35 | /** Type of space used to create chunk. */ 36 | public static final ConfigOption SPACE_TYPE = ConfigOptions 37 | .key("state.backend.spillable.space-type") 38 | .defaultValue(SpaceAllocator.SpaceType.MMAP.name()) 39 | .withDescription(String.format("Type of space used to create chunk. Options are %s (default), %s or %s.", 40 | SpaceAllocator.SpaceType.MMAP.name(), SpaceAllocator.SpaceType.HEAP.name(), SpaceAllocator.SpaceType.OFFHEAP.name())); 41 | 42 | /** Size of chunk. */ 43 | public static final ConfigOption CHUNK_SIZE = ConfigOptions 44 | .key("state.backend.spillable.chunk-size") 45 | .memoryType() 46 | .defaultValue(MemorySize.ofMebiBytes(512L)) 47 | .withDescription("Size of chunk which should be a power of two and no more than Integer#MAX_VALUE."); 48 | 49 | /** Maximum number of mmap files that can be used. */ 50 | public static final ConfigOption MAX_MMAP_FILES = ConfigOptions 51 | .key("state.backend.spillable.max-mmap-files") 52 | .intType() 53 | .defaultValue(Integer.MAX_VALUE) 54 | .withDescription("Maximum number of mmap files that can be used."); 55 | 56 | /** Interval to check heap status. */ 57 | public static final ConfigOption HEAP_STATUS_CHECK_INTERVAL = ConfigOptions 58 | .key("state.backend.spillable.heap-status.check-interval") 59 | .durationType() 60 | .defaultValue(Duration.ofMinutes(1L)) 61 | .withDescription("Interval to check heap status."); 62 | 63 | /** Threshold of gc time to trigger state spill. */ 64 | public static final ConfigOption GC_TIME_THRESHOLD = ConfigOptions 65 | .key("state.backend.spillable.gc-time.threshold") 66 | .durationType() 67 | .defaultValue(Duration.ofSeconds(2L)) 68 | .withDescription("If garbage collection time exceeds this threshold, state will be spilled."); 69 | 70 | /** Percentage of retained state size to spill in a turn. */ 71 | public static final ConfigOption SPILL_SIZE_RATIO = ConfigOptions 72 | .key("state.backend.spillable.spill-size.ratio") 73 | .floatType() 74 | .defaultValue(0.2f) 75 | .withDescription("Percentage of retained state size to spill in a turn."); 76 | 77 | /** State load will be triggered if memory usage is under this watermark. */ 78 | public static final ConfigOption LOAD_START_RATIO = ConfigOptions 79 | .key("state.backend.spillable.load-start.ratio") 80 | .floatType() 81 | .defaultValue(0.1f) 82 | .withDescription("State load will be triggered if memory usage is under this watermark."); 83 | 84 | /** Memory usage can't exceed this watermark after state load. */ 85 | public static final ConfigOption LOAD_END_RATIO = ConfigOptions 86 | .key("state.backend.spillable.load-end.ratio") 87 | .floatType() 88 | .defaultValue(0.3f) 89 | .withDescription("Memory usage can't exceed this watermark after state load."); 90 | 91 | /** Interval between continuous spill/load. */ 92 | public static final ConfigOption TRIGGER_INTERVAL = ConfigOptions 93 | .key("state.backend.spillable.trigger-interval") 94 | .durationType() 95 | .defaultValue(Duration.ofMinutes(1L)) 96 | .withDescription("Interval to trigger continuous spill/load."); 97 | 98 | /** Interval to check resource. */ 99 | public static final ConfigOption RESOURCE_CHECK_INTERVAL = ConfigOptions 100 | .key("state.backend.spillable.resource-check.interval") 101 | .durationType() 102 | .defaultValue(Duration.ofSeconds(10L)) 103 | .withDescription("Interval to check resource. High frequence will degrade performance but" 104 | + " be more sensitive to memory change."); 105 | 106 | /** Whether to cancel checkpoint before spill. */ 107 | public static final ConfigOption CANCEL_CHECKPOINT = ConfigOptions 108 | .key("state.backend.spillable.cancel.checkpoint") 109 | .booleanType() 110 | .defaultValue(true) 111 | .withDescription("Whether to cancel checkpoint before spill. Cancelling checkpoints will release " + 112 | "the spilled states and make them gc faster."); 113 | 114 | public static final ConfigOption[] SUPPORTED_CONFIG = new ConfigOption[]{ 115 | SPACE_TYPE, CHUNK_SIZE, MAX_MMAP_FILES, HEAP_STATUS_CHECK_INTERVAL, GC_TIME_THRESHOLD, 116 | SPILL_SIZE_RATIO, LOAD_START_RATIO, LOAD_END_RATIO, TRIGGER_INTERVAL, RESOURCE_CHECK_INTERVAL, CANCEL_CHECKPOINT 117 | }; 118 | 119 | public static Configuration filter(ReadableConfig readableConfig) { 120 | Configuration conf = new Configuration(); 121 | conf.set(SPACE_TYPE, readableConfig.get(SPACE_TYPE)); 122 | conf.set(CHUNK_SIZE, readableConfig.get(CHUNK_SIZE)); 123 | conf.set(MAX_MMAP_FILES, readableConfig.get(MAX_MMAP_FILES)); 124 | conf.set(HEAP_STATUS_CHECK_INTERVAL, readableConfig.get(HEAP_STATUS_CHECK_INTERVAL)); 125 | conf.set(GC_TIME_THRESHOLD, readableConfig.get(GC_TIME_THRESHOLD)); 126 | conf.set(SPILL_SIZE_RATIO, readableConfig.get(SPILL_SIZE_RATIO)); 127 | conf.set(LOAD_START_RATIO, readableConfig.get(LOAD_START_RATIO)); 128 | conf.set(LOAD_END_RATIO, readableConfig.get(LOAD_END_RATIO)); 129 | conf.set(TRIGGER_INTERVAL, readableConfig.get(TRIGGER_INTERVAL)); 130 | conf.set(RESOURCE_CHECK_INTERVAL, readableConfig.get(RESOURCE_CHECK_INTERVAL)); 131 | conf.set(CANCEL_CHECKPOINT, readableConfig.get(CANCEL_CHECKPOINT)); 132 | 133 | return conf; 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/main/java/org/apache/flink/runtime/state/heap/space/SpaceAllocator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.runtime.state.heap.space; 20 | 21 | import org.apache.flink.annotation.VisibleForTesting; 22 | import org.apache.flink.configuration.Configuration; 23 | import org.apache.flink.runtime.state.heap.SpillableOptions; 24 | import org.apache.flink.util.MathUtils; 25 | import org.apache.flink.util.Preconditions; 26 | 27 | import javax.annotation.Nullable; 28 | 29 | import java.io.File; 30 | import java.io.IOException; 31 | import java.util.ArrayList; 32 | import java.util.List; 33 | import java.util.concurrent.atomic.AtomicInteger; 34 | 35 | import static org.apache.flink.runtime.state.heap.space.SpaceConstants.BUCKET_SIZE; 36 | import static org.apache.flink.runtime.state.heap.space.SpaceConstants.FOUR_BYTES_BITS; 37 | import static org.apache.flink.runtime.state.heap.space.SpaceConstants.FOUR_BYTES_MARK; 38 | import static org.apache.flink.runtime.state.heap.space.SpaceConstants.NO_SPACE; 39 | 40 | /** 41 | * An implementation of {@link Allocator} which is not thread safe. 42 | */ 43 | public class SpaceAllocator implements Allocator { 44 | 45 | /** 46 | * Array of all chunks. 47 | */ 48 | private volatile Chunk[] totalSpace = new Chunk[16]; 49 | 50 | /** 51 | * List of chunks used to allocate space for data less than bucket size. 52 | */ 53 | private final List totalSpaceForNormal = new ArrayList<>(); 54 | 55 | /** 56 | * List of chunks used to allocate space for data bigger than or equal to bucket size. 57 | */ 58 | private final List totalSpaceForHuge = new ArrayList<>(); 59 | 60 | /** 61 | * Generator for chunk id. 62 | */ 63 | private final AtomicInteger chunkIdGenerator = new AtomicInteger(0); 64 | 65 | /** 66 | * The chunk allocator. 67 | */ 68 | private final ChunkAllocator chunkAllocator; 69 | 70 | public SpaceAllocator(Configuration configuration, @Nullable File[] localDirs) { 71 | this.chunkAllocator = createChunkAllocator(configuration, localDirs); 72 | } 73 | 74 | @VisibleForTesting 75 | SpaceAllocator(ChunkAllocator chunkAllocator) { 76 | this.chunkAllocator = Preconditions.checkNotNull(chunkAllocator); 77 | } 78 | 79 | @VisibleForTesting 80 | void addTotalSpace(Chunk chunk, int chunkId) { 81 | if (chunkId >= this.totalSpace.length) { 82 | Chunk[] chunkTemp = new Chunk[this.totalSpace.length * 2]; 83 | System.arraycopy(this.totalSpace, 0, chunkTemp, 0, this.totalSpace.length); 84 | this.totalSpace = chunkTemp; 85 | } 86 | totalSpace[chunkId] = chunk; 87 | } 88 | 89 | @Override 90 | public long allocate(int len) { 91 | if (len >= BUCKET_SIZE) { 92 | return doAllocate(totalSpaceForHuge, len, AllocateStrategy.HugeBucket); 93 | } else { 94 | return doAllocate(totalSpaceForNormal, len, AllocateStrategy.SmallBucket); 95 | } 96 | } 97 | 98 | @Override 99 | public void free(long offset) { 100 | int chunkId = SpaceUtils.getChunkIdByAddress(offset); 101 | int interChunkOffset = SpaceUtils.getChunkOffsetByAddress(offset); 102 | getChunkById(chunkId).free(interChunkOffset); 103 | } 104 | 105 | @Override 106 | public Chunk getChunkById(int chunkId) { 107 | return totalSpace[chunkId]; 108 | } 109 | 110 | private long doAllocate(List chunks, int len, AllocateStrategy allocateStrategy) { 111 | int offset; 112 | for (Chunk chunk : chunks) { 113 | offset = chunk.allocate(len); 114 | if (offset != NO_SPACE) { 115 | return ((chunk.getChunkId() & FOUR_BYTES_MARK) << FOUR_BYTES_BITS) | (offset & FOUR_BYTES_MARK); 116 | } 117 | } 118 | 119 | Chunk chunk = createChunk(allocateStrategy); 120 | chunks.add(chunk); 121 | addTotalSpace(chunk, chunk.getChunkId()); 122 | offset = chunk.allocate(len); 123 | 124 | if (offset != NO_SPACE) { 125 | return ((chunk.getChunkId() & FOUR_BYTES_MARK) << FOUR_BYTES_BITS) | (offset & FOUR_BYTES_MARK); 126 | } 127 | 128 | throw new RuntimeException("There is no space to allocate"); 129 | } 130 | 131 | private Chunk createChunk(AllocateStrategy allocateStrategy) { 132 | int chunkId = chunkIdGenerator.getAndIncrement(); 133 | return chunkAllocator.createChunk(chunkId, allocateStrategy); 134 | } 135 | 136 | @Override 137 | public void close() throws IOException { 138 | chunkAllocator.close(); 139 | } 140 | 141 | @VisibleForTesting 142 | ChunkAllocator getChunkAllocator() { 143 | return chunkAllocator; 144 | } 145 | 146 | @VisibleForTesting 147 | AtomicInteger getChunkIdGenerator() { 148 | return chunkIdGenerator; 149 | } 150 | 151 | 152 | /** 153 | * Type of space where to allocate chunks. 154 | */ 155 | public enum SpaceType { 156 | 157 | /** 158 | * Allocates chunks from heap. This is mainly used for test. 159 | */ 160 | HEAP, 161 | 162 | /** 163 | * Allocates chunks from off-heap. 164 | */ 165 | OFFHEAP, 166 | 167 | /** 168 | * Allocates chunks from mmp. 169 | */ 170 | MMAP 171 | } 172 | 173 | private static ChunkAllocator createChunkAllocator(Configuration configuration, @Nullable File[] localDirs) { 174 | SpaceType spaceType = SpaceType.valueOf(configuration.get(SpillableOptions.SPACE_TYPE).toUpperCase()); 175 | long chunkSize = configuration.get(SpillableOptions.CHUNK_SIZE).getBytes(); 176 | Preconditions.checkArgument(chunkSize <= Integer.MAX_VALUE, 177 | "Chunk size should be less than Integer.MAX_VALUE, but is actually %s", chunkSize); 178 | Preconditions.checkArgument(MathUtils.isPowerOf2(chunkSize), 179 | "Chunk size should be a power of two, but is actually %s", chunkSize); 180 | 181 | switch (spaceType) { 182 | case HEAP: 183 | return new HeapBufferChunkAllocator((int) chunkSize); 184 | case OFFHEAP: 185 | return new DirectBufferChunkAllocator((int) chunkSize); 186 | case MMAP: 187 | return new MmapChunkAllocator((int) chunkSize, localDirs, configuration); 188 | default: 189 | throw new UnsupportedOperationException("Unsupported space type " + spaceType.name()); 190 | } 191 | } 192 | } 193 | -------------------------------------------------------------------------------- /flink-statebackend-heap-spillable/src/test/java/org/apache/flink/runtime/state/heap/SpillableStateBackendFactoryTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * * Licensed to the Apache Software Foundation (ASF) under one 4 | * * or more contributor license agreements. See the NOTICE file 5 | * * distributed with this work for additional information 6 | * * regarding copyright ownership. The ASF licenses this file 7 | * * to you under the Apache License, Version 2.0 (the 8 | * * "License"); you may not use this file except in compliance 9 | * * with the License. You may obtain a copy of the License at 10 | * * 11 | * * http://www.apache.org/licenses/LICENSE-2.0 12 | * * 13 | * * Unless required by applicable law or agreed to in writing, software 14 | * * distributed under the License is distributed on an "AS IS" BASIS, 15 | * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | * * See the License for the specific language governing permissions and 17 | * * limitations under the License. 18 | * 19 | */ 20 | 21 | package org.apache.flink.runtime.state.heap; 22 | 23 | import org.apache.flink.configuration.CheckpointingOptions; 24 | import org.apache.flink.configuration.Configuration; 25 | import org.apache.flink.core.fs.Path; 26 | import org.apache.flink.runtime.state.StateBackend; 27 | import org.apache.flink.runtime.state.StateBackendLoader; 28 | import org.apache.flink.runtime.state.filesystem.AbstractFileStateBackend; 29 | import org.apache.flink.runtime.state.filesystem.FsStateBackend; 30 | import org.apache.flink.runtime.state.memory.MemoryStateBackend; 31 | 32 | import org.junit.Rule; 33 | import org.junit.Test; 34 | import org.junit.rules.TemporaryFolder; 35 | 36 | import static org.junit.Assert.assertEquals; 37 | import static org.junit.Assert.assertNull; 38 | import static org.junit.Assert.assertTrue; 39 | 40 | /** 41 | * Tests for the {@link SpillableStateBackendFactory}. 42 | */ 43 | public class SpillableStateBackendFactoryTest { 44 | 45 | @Rule 46 | public final TemporaryFolder tmp = new TemporaryFolder(); 47 | 48 | private final ClassLoader cl = getClass().getClassLoader(); 49 | 50 | private final String backendKey = CheckpointingOptions.STATE_BACKEND.key(); 51 | 52 | // ------------------------------------------------------------------------ 53 | 54 | @Test 55 | public void testFactoryName() { 56 | // construct the name such that it will not be automatically adjusted on refactorings 57 | String factoryName = "org.apache.flink.runtime.state.heap.Spill"; 58 | factoryName += "ableStateBackendFactory"; 59 | 60 | // !!! if this fails, the code in StateBackendLoader must be adjusted 61 | assertEquals(factoryName, SpillableStateBackendFactory.class.getName()); 62 | } 63 | 64 | /** 65 | * Validates loading a file system state backend with additional parameters from the cluster configuration. 66 | */ 67 | @Test 68 | public void testLoadFileSystemStateBackend() throws Exception { 69 | final String checkpointDir = new Path(tmp.newFolder().toURI()).toString(); 70 | final String savepointDir = new Path(tmp.newFolder().toURI()).toString(); 71 | 72 | final Path expectedCheckpointsPath = new Path(checkpointDir); 73 | final Path expectedSavepointsPath = new Path(savepointDir); 74 | 75 | final Configuration config = new Configuration(); 76 | config.setString(backendKey, SpillableStateBackendFactory.class.getName()); 77 | config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir); 78 | config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir); 79 | 80 | StateBackend backend = StateBackendLoader.loadStateBackendFromConfig(config, cl, null); 81 | 82 | assertTrue(backend instanceof SpillableStateBackend); 83 | 84 | SpillableStateBackend fs = (SpillableStateBackend) backend; 85 | 86 | AbstractFileStateBackend fsBackend = (AbstractFileStateBackend) fs.getCheckpointBackend(); 87 | 88 | assertTrue(fsBackend instanceof FsStateBackend); 89 | 90 | assertEquals(expectedCheckpointsPath, fsBackend.getCheckpointPath()); 91 | assertEquals(expectedSavepointsPath, fsBackend.getSavepointPath()); 92 | } 93 | 94 | /** 95 | * Validates loading a memory state backend with additional parameters from the cluster configuration. 96 | */ 97 | @Test 98 | public void testLoadMemoryStateBackend() throws Exception { 99 | final String savepointDir = new Path(tmp.newFolder().toURI()).toString(); 100 | 101 | final Path expectedSavepointsPath = new Path(savepointDir); 102 | 103 | final Configuration config = new Configuration(); 104 | config.setString(backendKey, SpillableStateBackendFactory.class.getName()); 105 | config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir); 106 | 107 | StateBackend backend = StateBackendLoader.loadStateBackendFromConfig(config, cl, null); 108 | 109 | assertTrue(backend instanceof SpillableStateBackend); 110 | 111 | SpillableStateBackend fs = (SpillableStateBackend) backend; 112 | 113 | AbstractFileStateBackend memBackend = (AbstractFileStateBackend) fs.getCheckpointBackend(); 114 | 115 | assertTrue(memBackend instanceof MemoryStateBackend); 116 | 117 | assertNull(memBackend.getCheckpointPath()); 118 | assertEquals(expectedSavepointsPath, memBackend.getSavepointPath()); 119 | } 120 | 121 | /** 122 | * Validates taking the application-defined file system state backend and adding with additional 123 | * parameters from the cluster configuration, but giving precedence to application-defined 124 | * parameters over configuration-defined parameters. 125 | */ 126 | @Test 127 | public void testLoadFileSystemStateBackendMixed() throws Exception { 128 | final String appCheckpointDir = new Path(tmp.newFolder().toURI()).toString(); 129 | final String checkpointDir = new Path(tmp.newFolder().toURI()).toString(); 130 | final String savepointDir = new Path(tmp.newFolder().toURI()).toString(); 131 | 132 | final Path expectedCheckpointsPath = new Path(appCheckpointDir); 133 | final Path expectedSavepointsPath = new Path(savepointDir); 134 | 135 | final SpillableStateBackend backend = new SpillableStateBackend(appCheckpointDir); 136 | 137 | final Configuration config = new Configuration(); 138 | config.setString(backendKey, "jobmanager"); // this should not be picked up 139 | config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir); // this should not be picked up 140 | config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir); 141 | 142 | final StateBackend loadedBackend = 143 | StateBackendLoader.fromApplicationOrConfigOrDefault(backend, config, cl, null); 144 | assertTrue(loadedBackend instanceof SpillableStateBackend); 145 | 146 | final SpillableStateBackend loadedRocks = (SpillableStateBackend) loadedBackend; 147 | 148 | AbstractFileStateBackend fsBackend = (AbstractFileStateBackend) loadedRocks.getCheckpointBackend(); 149 | assertEquals(expectedCheckpointsPath, fsBackend.getCheckpointPath()); 150 | assertEquals(expectedSavepointsPath, fsBackend.getSavepointPath()); 151 | } 152 | } 153 | --------------------------------------------------------------------------------