├── .mvn └── wrapper │ ├── maven-wrapper.jar │ └── maven-wrapper.properties ├── cassandra-2 ├── src │ ├── main │ │ ├── resources │ │ │ └── META-INF │ │ │ │ └── services │ │ │ │ └── com.instaclustr.cassandra.ttl.SSTableTTLRemover │ │ └── java │ │ │ └── com │ │ │ └── instaclustr │ │ │ └── cassandra │ │ │ └── ttl │ │ │ ├── NoTTLSerializer.java │ │ │ ├── NoTTLAbstractCell.java │ │ │ ├── NoTTLColumnSerializer.java │ │ │ ├── NoTTLSSTableIdentityIterator.java │ │ │ ├── Cassandra2TTLRemover.java │ │ │ ├── NoTTLSSTableNamesIterator.java │ │ │ └── NoTTLScanner.java │ └── test │ │ ├── resources │ │ └── cassandra │ │ │ └── cql │ │ │ ├── keyspace.cql │ │ │ └── table.cql │ │ └── java │ │ └── org │ │ └── apache │ │ └── cassandra │ │ └── ttl │ │ └── Cassandra2TTLRemoverTest.java └── pom.xml ├── cassandra-3 ├── src │ ├── main │ │ ├── resources │ │ │ └── META-INF │ │ │ │ └── services │ │ │ │ └── com.instaclustr.cassandra.ttl.SSTableTTLRemover │ │ └── java │ │ │ └── com │ │ │ └── instaclustr │ │ │ └── cassandra │ │ │ └── ttl │ │ │ └── Cassandra3TTLRemover.java │ └── test │ │ ├── resources │ │ └── cassandra │ │ │ └── cql │ │ │ ├── keyspace.cql │ │ │ └── table.cql │ │ └── java │ │ └── org │ │ └── apache │ │ └── cassandra │ │ └── ttl │ │ └── Cassandra3TTLRemoverTest.java └── pom.xml ├── cassandra-4 ├── src │ ├── main │ │ ├── resources │ │ │ └── META-INF │ │ │ │ └── services │ │ │ │ └── com.instaclustr.cassandra.ttl.SSTableTTLRemover │ │ └── java │ │ │ └── com │ │ │ └── instaclustr │ │ │ └── cassandra │ │ │ └── ttl │ │ │ └── Cassandra4TTLRemover.java │ └── test │ │ ├── resources │ │ └── cassandra │ │ │ └── cql │ │ │ ├── keyspace.cql │ │ │ └── table.cql │ │ └── java │ │ └── com │ │ └── instaclustr │ │ └── cassandra │ │ └── ttl │ │ └── Cassandra4TTLRemoverTest.java └── pom.xml ├── cassandra-4.1 ├── src │ ├── main │ │ ├── resources │ │ │ └── META-INF │ │ │ │ └── services │ │ │ │ └── com.instaclustr.cassandra.ttl.SSTableTTLRemover │ │ └── java │ │ │ └── com │ │ │ └── instaclustr │ │ │ └── cassandra │ │ │ └── ttl │ │ │ └── Cassandra41TTLRemover.java │ └── test │ │ ├── resources │ │ └── cassandra │ │ │ └── cql │ │ │ ├── keyspace.cql │ │ │ └── table.cql │ │ └── java │ │ └── com │ │ └── instaclustr │ │ └── cassandra │ │ └── ttl │ │ └── Cassandra41TTLRemoverTest.java └── pom.xml ├── impl ├── src │ └── main │ │ └── java │ │ └── com │ │ └── instaclustr │ │ └── cassandra │ │ └── ttl │ │ ├── SSTableTTLRemover.java │ │ └── cli │ │ ├── TTLRemovalException.java │ │ ├── JarManifestVersionProvider.java │ │ └── TTLRemoverCLI.java └── pom.xml ├── .gitignore ├── .github └── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── .circleci └── config.yml ├── buddy-agent ├── pom.xml └── src │ └── main │ └── java │ └── com │ └── instaclustr │ └── cassandra │ └── ttl │ └── buddy │ └── CassandraAgent.java ├── run.sh ├── pom.xml ├── mvnw └── README.adoc /.mvn/wrapper/maven-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instaclustr/cassandra-ttl-remover/HEAD/.mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /cassandra-2/src/main/resources/META-INF/services/com.instaclustr.cassandra.ttl.SSTableTTLRemover: -------------------------------------------------------------------------------- 1 | com.instaclustr.cassandra.ttl.Cassandra2TTLRemover -------------------------------------------------------------------------------- /cassandra-3/src/main/resources/META-INF/services/com.instaclustr.cassandra.ttl.SSTableTTLRemover: -------------------------------------------------------------------------------- 1 | com.instaclustr.cassandra.ttl.Cassandra3TTLRemover -------------------------------------------------------------------------------- /cassandra-4/src/main/resources/META-INF/services/com.instaclustr.cassandra.ttl.SSTableTTLRemover: -------------------------------------------------------------------------------- 1 | com.instaclustr.cassandra.ttl.Cassandra4TTLRemover -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.6.0/apache-maven-3.6.0-bin.zip 2 | -------------------------------------------------------------------------------- /cassandra-4.1/src/main/resources/META-INF/services/com.instaclustr.cassandra.ttl.SSTableTTLRemover: -------------------------------------------------------------------------------- 1 | com.instaclustr.cassandra.ttl.Cassandra41TTLRemover -------------------------------------------------------------------------------- /cassandra-2/src/test/resources/cassandra/cql/keyspace.cql: -------------------------------------------------------------------------------- 1 | CREATE KEYSPACE IF NOT EXISTS test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1 }; -------------------------------------------------------------------------------- /cassandra-3/src/test/resources/cassandra/cql/keyspace.cql: -------------------------------------------------------------------------------- 1 | CREATE KEYSPACE IF NOT EXISTS test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1 }; -------------------------------------------------------------------------------- /cassandra-4.1/src/test/resources/cassandra/cql/keyspace.cql: -------------------------------------------------------------------------------- 1 | CREATE KEYSPACE IF NOT EXISTS test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1 }; -------------------------------------------------------------------------------- /cassandra-4/src/test/resources/cassandra/cql/keyspace.cql: -------------------------------------------------------------------------------- 1 | CREATE KEYSPACE IF NOT EXISTS test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1 }; -------------------------------------------------------------------------------- /cassandra-2/src/test/resources/cassandra/cql/table.cql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS test.test (id uuid, name text, surname text, PRIMARY KEY (id)) WITH default_time_to_live = 10; -------------------------------------------------------------------------------- /cassandra-3/src/test/resources/cassandra/cql/table.cql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS test.test (id uuid, name text, surname text, PRIMARY KEY (id)) WITH default_time_to_live = 10; -------------------------------------------------------------------------------- /cassandra-4.1/src/test/resources/cassandra/cql/table.cql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS test.test (id uuid, name text, surname text, PRIMARY KEY (id)) WITH default_time_to_live = 10; -------------------------------------------------------------------------------- /cassandra-4/src/test/resources/cassandra/cql/table.cql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS test.test (id uuid, name text, surname text, PRIMARY KEY (id)) WITH default_time_to_live = 10; -------------------------------------------------------------------------------- /impl/src/main/java/com/instaclustr/cassandra/ttl/SSTableTTLRemover.java: -------------------------------------------------------------------------------- 1 | package com.instaclustr.cassandra.ttl; 2 | 3 | import java.nio.file.Path; 4 | import java.util.Collection; 5 | 6 | public interface SSTableTTLRemover { 7 | 8 | void executeRemoval(final Path outputFolder, final Collection sstables, final String cql) throws Exception; 9 | } 10 | -------------------------------------------------------------------------------- /impl/src/main/java/com/instaclustr/cassandra/ttl/cli/TTLRemovalException.java: -------------------------------------------------------------------------------- 1 | package com.instaclustr.cassandra.ttl.cli; 2 | 3 | public class TTLRemovalException extends Exception { 4 | 5 | public TTLRemovalException() { 6 | } 7 | 8 | public TTLRemovalException(String message) { 9 | super(message); 10 | } 11 | 12 | public TTLRemovalException(String message, Throwable cause) { 13 | super(message, cause); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | /bin/ 3 | 4 | # Java class files 5 | *.class 6 | 7 | # generated files 8 | gen/ 9 | 10 | # Local configuration file (sdk path, etc) 11 | local.properties 12 | 13 | # Eclipse stuff 14 | .classpath 15 | .project 16 | .settings 17 | 18 | # Gradle stuff 19 | .gradle/ 20 | build/ 21 | .idea/ 22 | out/ 23 | *.iml 24 | 25 | # NetBeans project 26 | **/.nb-gradle/ 27 | **/.nb-gradle-properties 28 | *.pyc 29 | 30 | *.swp 31 | *.log 32 | 33 | dependency-reduced-pom.xml 34 | 35 | cassandra.logdir_IS_UNDEFINED 36 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve TTL remover 4 | title: "[BUG]" 5 | labels: bug 6 | assignees: smiklosovic 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 16 | **Expected behavior** 17 | A clear and concise description of what you expected to happen. 18 | 19 | **Desktop (please complete the following information):** 20 | - OS, Cassandra version, Java version, TTL remover version 21 | 22 | **Additional context** 23 | Add any other context about the problem here. 24 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: "[FEATURE]" 5 | labels: feature 6 | assignees: smiklosovic 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | jobs: 3 | build: 4 | 5 | working_directory: ~/cassandra-ttl-remover 6 | 7 | docker: 8 | - image: cimg/openjdk:8.0 9 | 10 | steps: 11 | 12 | - checkout 13 | 14 | - restore_cache: 15 | keys: 16 | - m2-{{ checksum "pom.xml" }} 17 | 18 | - save_cache: 19 | paths: 20 | - ~/.m2 21 | key: m2-{{ checksum "pom.xml" }} 22 | 23 | - run: mvn install -Dversion.cassandra2=2.2.19 -Dversion.cassandra3=3.11.14 -Dversion.cassandra4=4.0.7 -Dversion.cassandra41=4.1.0 24 | 25 | - persist_to_workspace: 26 | root: ~/cassandra-ttl-remover 27 | paths: 28 | - "cassandra-2/target/cassandra-ldap-2.jar" 29 | - "cassandra-3/target/cassandra-ldap-3.jar" 30 | - "cassandra-4/target/ttl-remover-cassandra-4.jar" 31 | - "cassandra-4.1/target/ttl-remover-cassandra-4.1.jar" 32 | -------------------------------------------------------------------------------- /cassandra-2/src/main/java/com/instaclustr/cassandra/ttl/NoTTLSerializer.java: -------------------------------------------------------------------------------- 1 | package com.instaclustr.cassandra.ttl; 2 | 3 | import java.io.DataInput; 4 | import java.io.IOException; 5 | 6 | import org.apache.cassandra.db.ColumnSerializer; 7 | import org.apache.cassandra.db.OnDiskAtom; 8 | import org.apache.cassandra.db.composites.CellName; 9 | import org.apache.cassandra.db.composites.CellNameType; 10 | import org.apache.cassandra.db.composites.Composite; 11 | import org.apache.cassandra.io.sstable.format.Version; 12 | 13 | public class NoTTLSerializer extends OnDiskAtom.Serializer { 14 | 15 | private final CellNameType type; 16 | 17 | public NoTTLSerializer(CellNameType type) { 18 | super(type); 19 | this.type = type; 20 | } 21 | 22 | @Override 23 | public OnDiskAtom deserializeFromSSTable(DataInput in, ColumnSerializer.Flag flag, int expireBefore, Version version) throws IOException { 24 | Composite name = type.serializer().deserialize(in); 25 | if (name.isEmpty()) { 26 | // SSTableWriter.END_OF_ROW 27 | return null; 28 | } 29 | 30 | int b = in.readUnsignedByte(); 31 | if ((b & ColumnSerializer.RANGE_TOMBSTONE_MASK) != 0) { 32 | return type.rangeTombstoneSerializer().deserializeBody(in, name, version); 33 | } else { 34 | return new NoTTLColumnSerializer(type).deserializeColumnBody(in, (CellName) name, b, flag, expireBefore); 35 | } 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /cassandra-2/src/main/java/com/instaclustr/cassandra/ttl/NoTTLAbstractCell.java: -------------------------------------------------------------------------------- 1 | package com.instaclustr.cassandra.ttl; 2 | 3 | import java.io.DataInput; 4 | import java.io.IOError; 5 | import java.io.IOException; 6 | import java.util.Iterator; 7 | 8 | import com.google.common.collect.AbstractIterator; 9 | import org.apache.cassandra.db.AbstractCell; 10 | import org.apache.cassandra.db.ColumnSerializer; 11 | import org.apache.cassandra.db.OnDiskAtom; 12 | import org.apache.cassandra.db.composites.CellNameType; 13 | import org.apache.cassandra.io.sstable.format.Version; 14 | 15 | public abstract class NoTTLAbstractCell extends AbstractCell 16 | { 17 | public static Iterator onDiskIterator(final DataInput in, 18 | final ColumnSerializer.Flag flag, 19 | final int expireBefore, 20 | final Version version, 21 | final CellNameType type) 22 | { 23 | return new AbstractIterator() 24 | { 25 | protected OnDiskAtom computeNext() 26 | { 27 | OnDiskAtom atom; 28 | try 29 | { 30 | atom = new NoTTLSerializer(type).deserializeFromSSTable(in, flag, expireBefore, version); 31 | } 32 | catch (IOException e) 33 | { 34 | throw new IOError(e); 35 | } 36 | if (atom == null) 37 | return endOfData(); 38 | 39 | return atom; 40 | } 41 | }; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /cassandra-2/src/main/java/com/instaclustr/cassandra/ttl/NoTTLColumnSerializer.java: -------------------------------------------------------------------------------- 1 | package com.instaclustr.cassandra.ttl; 2 | 3 | import java.io.DataInput; 4 | import java.io.IOException; 5 | import java.nio.ByteBuffer; 6 | 7 | import org.apache.cassandra.db.BufferCell; 8 | import org.apache.cassandra.db.BufferCounterCell; 9 | import org.apache.cassandra.db.BufferCounterUpdateCell; 10 | import org.apache.cassandra.db.BufferDeletedCell; 11 | import org.apache.cassandra.db.Cell; 12 | import org.apache.cassandra.db.ColumnSerializer; 13 | import org.apache.cassandra.db.composites.CellName; 14 | import org.apache.cassandra.db.composites.CellNameType; 15 | import org.apache.cassandra.utils.ByteBufferUtil; 16 | 17 | public class NoTTLColumnSerializer extends ColumnSerializer 18 | { 19 | public NoTTLColumnSerializer(CellNameType type) 20 | { 21 | super(type); 22 | } 23 | 24 | Cell deserializeColumnBody(DataInput in, CellName name, int mask, Flag flag, int expireBefore) throws IOException 25 | { 26 | if ((mask & COUNTER_MASK) != 0) 27 | { 28 | long timestampOfLastDelete = in.readLong(); 29 | long ts = in.readLong(); 30 | ByteBuffer value = ByteBufferUtil.readWithLength(in); 31 | return BufferCounterCell.create(name, value, ts, timestampOfLastDelete, flag); 32 | } 33 | else if ((mask & EXPIRATION_MASK) != 0) 34 | { 35 | int ttl = in.readInt(); 36 | int expiration = in.readInt(); 37 | long ts = in.readLong(); 38 | ByteBuffer value = ByteBufferUtil.readWithLength(in); 39 | return new BufferCell(name, value, ts); 40 | } 41 | else 42 | { 43 | long ts = in.readLong(); 44 | ByteBuffer value = ByteBufferUtil.readWithLength(in); 45 | return (mask & COUNTER_UPDATE_MASK) != 0 46 | ? new BufferCounterUpdateCell(name, value, ts) 47 | : ((mask & DELETION_MASK) == 0 48 | ? new BufferCell(name, value, ts) 49 | : new BufferDeletedCell(name, value, ts)); 50 | } 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /impl/src/main/java/com/instaclustr/cassandra/ttl/cli/JarManifestVersionProvider.java: -------------------------------------------------------------------------------- 1 | package com.instaclustr.cassandra.ttl.cli; 2 | 3 | import static java.lang.String.format; 4 | 5 | import java.io.IOException; 6 | import java.net.URL; 7 | import java.util.Enumeration; 8 | import java.util.Optional; 9 | import java.util.jar.Attributes; 10 | import java.util.jar.Manifest; 11 | 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | import picocli.CommandLine; 15 | import picocli.CommandLine.IVersionProvider; 16 | 17 | public abstract class JarManifestVersionProvider implements IVersionProvider { 18 | 19 | @Override 20 | public String[] getVersion() throws IOException { 21 | final Enumeration resources = CommandLine.class.getClassLoader().getResources("META-INF/MANIFEST.MF"); 22 | 23 | Optional implementationVersion = Optional.empty(); 24 | Optional buildTime = Optional.empty(); 25 | Optional gitCommit = Optional.empty(); 26 | 27 | while (resources.hasMoreElements()) { 28 | final URL url = resources.nextElement(); 29 | 30 | final Manifest manifest = new Manifest(url.openStream()); 31 | final Attributes attributes = manifest.getMainAttributes(); 32 | 33 | if (isApplicableManifest(attributes)) { 34 | implementationVersion = Optional.ofNullable(attributes.getValue(Attributes.Name.IMPLEMENTATION_VERSION)); 35 | buildTime = Optional.ofNullable(attributes.getValue("Build-Time")); 36 | gitCommit = Optional.ofNullable(attributes.getValue("Git-Commit")); 37 | 38 | break; 39 | } 40 | } 41 | 42 | return new String[]{ 43 | format("%s %s", getImplementationTitle(), implementationVersion.orElse("development build")), 44 | format("Build time: %s", buildTime.orElse("unknown")), 45 | format("Git commit: %s", gitCommit.orElse("unknown")), 46 | }; 47 | } 48 | 49 | private boolean isApplicableManifest(Attributes attributes) { 50 | return getImplementationTitle().equals(attributes.getValue(Attributes.Name.IMPLEMENTATION_TITLE)); 51 | } 52 | 53 | public abstract String getImplementationTitle(); 54 | 55 | public static void logCommandVersionInformation(final CommandLine.Model.CommandSpec commandSpec) { 56 | final Logger logger = LoggerFactory.getLogger(commandSpec.userObject().getClass()); 57 | logger.info("{} version: {}", commandSpec.name(), String.join(", ", commandSpec.version())); 58 | } 59 | } 60 | 61 | -------------------------------------------------------------------------------- /buddy-agent/pom.xml: -------------------------------------------------------------------------------- 1 | 4 | 4.0.0 5 | 6 | 7 | com.instaclustr 8 | ttl-remover-parent 9 | 1.1.3-SNAPSHOT 10 | ../pom.xml 11 | 12 | 13 | ttl-remover-byte-buddy 14 | 1.0.4 15 | 16 | 17 | 4.0.0 18 | 1.10.16 19 | 20 | 3.1.1 21 | 3.2.1 22 | false 23 | 24 | UTF-8 25 | 26 | 1.8 27 | 1.8 28 | 29 | 30 | 31 | 32 | net.bytebuddy 33 | byte-buddy 34 | ${version.bytebuddy} 35 | 36 | 37 | 38 | org.apache.cassandra 39 | cassandra-all 40 | ${version.cassandra} 41 | provided 42 | 43 | 44 | 45 | 46 | byte-buddy-agent 47 | 48 | 49 | 50 | org.apache.maven.plugins 51 | maven-jar-plugin 52 | ${maven.jar.plugin.version} 53 | 54 | 55 | 56 | true 57 | true 58 | 59 | 60 | com.instaclustr.cassandra.ttl.buddy.CassandraAgent 61 | 62 | https://github.com/instaclustr/TTLRemover/tree/${project.version}/buddy-agent 63 | 64 | ${git.commit.id} 65 | ${git.build.time} 66 | 67 | 68 | 69 | 70 | 71 | org.apache.maven.plugins 72 | maven-shade-plugin 73 | ${maven.shade.plugin.version} 74 | 75 | false 76 | 77 | 78 | 79 | package 80 | 81 | shade 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | if [ "x$CASSANDRA_HOME" != "x" ]; then 20 | CASSANDRA_INCLUDE=$CASSANDRA_HOME/bin/cassandra.in.sh 21 | fi 22 | 23 | if [ "x$CASSANDRA_INCLUDE" = "x" ]; then 24 | for include in "`dirname "$0"`/cassandra.in.sh" \ 25 | "$HOME/.cassandra.in.sh" \ 26 | /usr/share/cassandra/cassandra.in.sh \ 27 | /usr/local/share/cassandra/cassandra.in.sh \ 28 | /opt/cassandra/cassandra.in.sh; do 29 | if [ -r "$include" ]; then 30 | . "$include" 31 | break 32 | fi 33 | done 34 | elif [ -r "$CASSANDRA_INCLUDE" ]; then 35 | . "$CASSANDRA_INCLUDE" 36 | fi 37 | 38 | 39 | # Use JAVA_HOME if set, otherwise look for java in PATH 40 | if [ -x "$JAVA_HOME/bin/java" ]; then 41 | JAVA="$JAVA_HOME/bin/java" 42 | else 43 | JAVA="`which java`" 44 | fi 45 | 46 | if [ -z "$CLASSPATH" ]; then 47 | echo "You must set the CLASSPATH var" >&2 48 | exit 1 49 | fi 50 | 51 | set +x 52 | 53 | # For Cassandra 4.1 54 | 55 | java -Dcassandra.storagedir=$CASSANDRA_HOME/data -Dcassandra.config=file:///$CASSANDRA_HOME/conf/cassandra.yaml \ 56 | -cp "$CLASSPATH./impl/target/ttl-remover.jar:./cassandra-4.1/target/ttl-remover-cassandra-4.1.jar" \ 57 | $JVM_OPTS \ 58 | com.instaclustr.cassandra.ttl.cli.TTLRemoverCLI \ 59 | --cassandra-version=4 \ 60 | --sstables \ 61 | /tmp/original-4/test/test \ 62 | --output-path \ 63 | /tmp/stripped \ 64 | --cql \ 65 | 'CREATE TABLE IF NOT EXISTS test.test (id uuid, name text, surname text, PRIMARY KEY (id)) WITH default_time_to_live = 10;' 66 | 67 | # For Cassandra 3 and 4.0 68 | 69 | # change versions of jars on classpath to target 3 or 4 70 | # change --cassandra-version if necessary 71 | #java -javaagent:./buddy-agent/target/byte-buddy-agent.jar \ 72 | # -cp "$CLASSPATH./impl/target/ttl-remover.jar:./cassandra-3/target/ttl-remover-cassandra-3.jar" \ 73 | # $JVM_OPTS \ 74 | # com.instaclustr.cassandra.ttl.cli.TTLRemoverCLI \ 75 | # --cassandra-version=3 \ 76 | # --sstables \ 77 | # /tmp/original-3/test/test \ 78 | # --output-path \ 79 | # /tmp/stripped \ 80 | # --cql \ 81 | # 'CREATE TABLE IF NOT EXISTS test.test (id uuid, name text, surname text, PRIMARY KEY (id)) WITH default_time_to_live = 10;' 82 | 83 | # For Cassandra 2 84 | 85 | #java -cp "$CLASSPATH./impl/target/ttl-remover.jar:./cassandra-2/target/ttl-remover-cassandra-2.jar" $JVM_OPTS \ 86 | # com.instaclustr.cassandra.ttl.cli.TTLRemoverCLI \ 87 | # --cassandra-version=2 \ 88 | # --sstables \ 89 | # /tmp/sstables2/test \ 90 | # --output-path \ 91 | # /tmp/stripped \ 92 | # --cassandra-yaml \ 93 | # $CASSANDRA_HOME/conf/cassandra.yaml \ 94 | # --cassandra-storage-dir \ 95 | # $CASSANDRA_HOME/data 96 | -------------------------------------------------------------------------------- /cassandra-3/pom.xml: -------------------------------------------------------------------------------- 1 | 4 | 4.0.0 5 | 6 | 7 | com.instaclustr 8 | ttl-remover-parent 9 | 1.1.3-SNAPSHOT 10 | ../pom.xml 11 | 12 | 13 | ttl-remover-cassandra-3-11-14 14 | 1.0 15 | 16 | 17 | 3.11.14 18 | 3.0.2 19 | 1.5 20 | 4.0.3 21 | 4.13.1 22 | 3.0.1 23 | 24 | 1.0.1 25 | 3.1.1 26 | 2.2.4 27 | 3.2.1 28 | 29 | UTF-8 30 | 31 | 1.8 32 | 1.8 33 | 34 | 35 | 36 | 37 | com.instaclustr 38 | ttl-remover-impl 39 | 1.1.3-SNAPSHOT 40 | provided 41 | 42 | 43 | 44 | org.apache.cassandra 45 | cassandra-all 46 | ${version.cassandra3} 47 | provided 48 | 49 | 50 | 51 | com.instaclustr 52 | sstable-generator 53 | ${version.generator} 54 | test 55 | 56 | 57 | 58 | com.instaclustr 59 | sstable-generator-cassandra-3-11-14 60 | 1.0 61 | test 62 | 63 | 64 | com.datastax.oss 65 | java-driver-core 66 | 67 | 68 | 69 | 70 | 71 | com.github.nosan 72 | embedded-cassandra 73 | ${version.embedded.cassandra} 74 | test 75 | 76 | 77 | com.datastax.oss 78 | java-driver-core 79 | 80 | 81 | 82 | 83 | 84 | org.awaitility 85 | awaitility 86 | ${version.awaitility} 87 | test 88 | 89 | 90 | 91 | junit 92 | junit 93 | ${junit.version} 94 | test 95 | 96 | 97 | 98 | 99 | ttl-remover-cassandra-3 100 | 101 | 102 | -------------------------------------------------------------------------------- /cassandra-2/pom.xml: -------------------------------------------------------------------------------- 1 | 4 | 4.0.0 5 | 6 | 7 | com.instaclustr 8 | ttl-remover-parent 9 | 1.1.3-SNAPSHOT 10 | ../pom.xml 11 | 12 | 13 | ttl-remover-cassandra-2-2-19 14 | 1.0 15 | 16 | 17 | 2.2.19 18 | 19 | 3.0.2 20 | 1.5 21 | 4.0.3 22 | 4.13.1 23 | 3.0.1 24 | 25 | 1.0.1 26 | 3.1.1 27 | 2.2.4 28 | 3.2.1 29 | 30 | false 31 | 32 | UTF-8 33 | 34 | 1.8 35 | 1.8 36 | 37 | 38 | 39 | 40 | com.instaclustr 41 | ttl-remover-impl 42 | 1.1.3-SNAPSHOT 43 | provided 44 | 45 | 46 | 47 | org.apache.cassandra 48 | cassandra-all 49 | ${version.cassandra2} 50 | provided 51 | 52 | 53 | 54 | com.google.guava 55 | guava 56 | 16.0.1 57 | 58 | 59 | 60 | com.instaclustr 61 | sstable-generator 62 | ${version.generator} 63 | test 64 | 65 | 66 | 67 | com.instaclustr 68 | sstable-generator-cassandra-2-2-19 69 | 1.0 70 | test 71 | 72 | 73 | com.datastax.oss 74 | java-driver-core 75 | 76 | 77 | 78 | 79 | 80 | com.github.nosan 81 | embedded-cassandra 82 | ${version.embedded.cassandra} 83 | test 84 | 85 | 86 | com.datastax.oss 87 | java-driver-core 88 | 89 | 90 | 91 | 92 | 93 | org.awaitility 94 | awaitility 95 | ${version.awaitility} 96 | test 97 | 98 | 99 | 100 | junit 101 | junit 102 | ${junit.version} 103 | test 104 | 105 | 106 | 107 | 108 | ttl-remover-cassandra-2 109 | 110 | 111 | -------------------------------------------------------------------------------- /cassandra-4/pom.xml: -------------------------------------------------------------------------------- 1 | 4 | 4.0.0 5 | 6 | 7 | com.instaclustr 8 | ttl-remover-parent 9 | 1.1.3-SNAPSHOT 10 | ../pom.xml 11 | 12 | 13 | ttl-remover-cassandra-4.0.7 14 | 1.0 15 | 16 | 17 | 4.0.7 18 | 19 | 4.0.1 20 | 1.5 21 | 4.0.3 22 | 4.13.1 23 | 3.11.0 24 | 25 | 1.0.1 26 | 3.1.1 27 | 2.2.4 28 | 3.2.1 29 | 30 | UTF-8 31 | 32 | 1.8 33 | 1.8 34 | 35 | 36 | 37 | 38 | com.instaclustr 39 | ttl-remover-impl 40 | 1.1.3-SNAPSHOT 41 | provided 42 | 43 | 44 | 45 | org.apache.cassandra 46 | cassandra-all 47 | ${version.cassandra4} 48 | provided 49 | 50 | 51 | 52 | 53 | 54 | com.instaclustr 55 | sstable-generator 56 | ${version.generator} 57 | test 58 | 59 | 60 | 61 | com.instaclustr 62 | sstable-generator-cassandra-4-0-7 63 | 1.0 64 | test 65 | 66 | 67 | com.datastax.oss 68 | java-driver-core 69 | 70 | 71 | 72 | 73 | 74 | com.github.nosan 75 | embedded-cassandra 76 | ${version.embedded.cassandra} 77 | test 78 | 79 | 80 | com.datastax.oss 81 | java-driver-core 82 | 83 | 84 | 85 | 86 | 87 | org.awaitility 88 | awaitility 89 | ${version.awaitility} 90 | test 91 | 92 | 93 | 94 | junit 95 | junit 96 | ${junit.version} 97 | test 98 | 99 | 100 | 101 | 102 | ttl-remover-cassandra-4 103 | 104 | 105 | 106 | -------------------------------------------------------------------------------- /cassandra-4.1/pom.xml: -------------------------------------------------------------------------------- 1 | 4 | 4.0.0 5 | 6 | 7 | com.instaclustr 8 | ttl-remover-parent 9 | 1.1.3-SNAPSHOT 10 | ../pom.xml 11 | 12 | 13 | ttl-remover-cassandra-4.1.0 14 | 1.0 15 | 16 | 17 | 4.1.0 18 | 19 | 4.0.1 20 | 1.5 21 | 4.0.3 22 | 4.13.1 23 | 3.11.0 24 | 25 | 1.0.1 26 | 3.1.1 27 | 2.2.4 28 | 3.2.1 29 | 30 | UTF-8 31 | 32 | 1.8 33 | 1.8 34 | 35 | 36 | 37 | 38 | com.instaclustr 39 | ttl-remover-impl 40 | 1.1.3-SNAPSHOT 41 | provided 42 | 43 | 44 | 45 | org.apache.cassandra 46 | cassandra-all 47 | ${version.cassandra41} 48 | provided 49 | 50 | 51 | 52 | 53 | 54 | com.instaclustr 55 | sstable-generator 56 | ${version.generator} 57 | test 58 | 59 | 60 | 61 | com.instaclustr 62 | sstable-generator-cassandra-4-1-0 63 | 1.0 64 | test 65 | 66 | 67 | com.datastax.oss 68 | java-driver-core 69 | 70 | 71 | 72 | 73 | 74 | com.github.nosan 75 | embedded-cassandra 76 | ${version.embedded.cassandra} 77 | test 78 | 79 | 80 | com.datastax.oss 81 | java-driver-core 82 | 83 | 84 | 85 | 86 | 87 | org.awaitility 88 | awaitility 89 | ${version.awaitility} 90 | test 91 | 92 | 93 | 94 | junit 95 | junit 96 | ${junit.version} 97 | test 98 | 99 | 100 | 101 | 102 | ttl-remover-cassandra-4.1 103 | 104 | 105 | 106 | -------------------------------------------------------------------------------- /impl/pom.xml: -------------------------------------------------------------------------------- 1 | 4 | 4.0.0 5 | 6 | 7 | com.instaclustr 8 | ttl-remover-parent 9 | 1.1.3-SNAPSHOT 10 | ../pom.xml 11 | 12 | 13 | ttl-remover-impl 14 | 1.1.3-SNAPSHOT 15 | 16 | 17 | 1.7.30 18 | 4.5.1 19 | 3.0.1 20 | 21 | 3.0.2 22 | 3.1.6 23 | 4.12 24 | 25 | 1.0.1 26 | 3.1.1 27 | 2.2.4 28 | 3.2.1 29 | 30 | UTF-8 31 | 32 | 1.8 33 | 1.8 34 | 35 | 36 | 37 | 38 | info.picocli 39 | picocli 40 | ${picocli.version} 41 | 42 | 43 | 44 | org.slf4j 45 | slf4j-api 46 | ${slf4j.version} 47 | 48 | 49 | 50 | 51 | ttl-remover 52 | 53 | 54 | org.apache.maven.plugins 55 | maven-jar-plugin 56 | ${maven.jar.plugin.version} 57 | 58 | 59 | 60 | true 61 | true 62 | 63 | 64 | ${git.commit.id} 65 | ${git.build.time} 66 | 67 | 68 | 69 | 70 | 71 | pl.project13.maven 72 | git-commit-id-plugin 73 | ${git.command.plugin.version} 74 | 75 | 76 | 77 | revision 78 | 79 | 80 | 81 | 82 | ${project.basedir}/../.git 83 | 84 | 85 | 86 | org.apache.maven.plugins 87 | maven-shade-plugin 88 | ${maven.shade.plugin.version} 89 | 90 | false 91 | true 92 | 93 | ${java.io.tmpdir}/dependency-reduced-pom.xml 94 | 95 | 96 | 98 | com.instaclustr.cassandra.ttl.cli.TTLRemoverCLI 99 | 100 | 101 | 102 | 103 | 104 | package 105 | 106 | shade 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 4 | 4.0.0 5 | 6 | com.instaclustr 7 | ttl-remover-parent 8 | 1.1.3-SNAPSHOT 9 | pom 10 | 11 | 12 | buddy-agent 13 | cassandra-2 14 | cassandra-3 15 | cassandra-4 16 | cassandra-4.1 17 | impl 18 | 19 | 20 | ttl-remover-parent 21 | Tool for removing TTL information from Cassandra SSTables 22 | https://github.com/instaclustr/cassandra-ttl-remover 23 | 24 | 2020 25 | 26 | 27 | 28 | The Apache License, Version 2.0 29 | https://www.apache.org/licenses/LICENSE-2.0.txt 30 | 31 | 32 | 33 | 34 | 35 | Various 36 | Instaclustr 37 | https://www.instaclustr.com 38 | 39 | 40 | 41 | 42 | Instaclustr 43 | https://instaclustr.com 44 | 45 | 46 | 47 | scm:git:git://git@github.com:instaclustr/cassandra-ttl-remover.git 48 | scm:git:ssh://github.com/instaclustr/cassandra-ttl-remover.git 49 | git://github.com/instaclustr/cassandra-ttl-remover.git 50 | 51 | 52 | 53 | 54 | ossrh 55 | https://oss.sonatype.org/service/local/staging/deploy/maven2/ 56 | 57 | 58 | 59 | 60 | 3.1.1 61 | 3.1.0 62 | 1.6 63 | 2.8.2 64 | 1.6.8 65 | 66 | UTF-8 67 | 68 | 1.8 69 | 1.8 70 | 71 | 72 | 73 | 74 | 75 | org.apache.maven.plugins 76 | maven-deploy-plugin 77 | ${maven.deploy.plugin.version} 78 | 79 | true 80 | 81 | 82 | 83 | org.apache.maven.plugins 84 | maven-javadoc-plugin 85 | ${maven.javadoc.plugin.version} 86 | 87 | 88 | attach-javadocs 89 | 90 | jar 91 | 92 | 93 | 94 | 95 | 96 | org.apache.maven.plugins 97 | maven-source-plugin 98 | ${maven.source.plugin.version} 99 | 100 | 101 | attach-sources 102 | 103 | jar-no-fork 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | release 115 | 116 | 117 | 118 | org.apache.maven.plugins 119 | maven-gpg-plugin 120 | ${maven.gpg.plugin.version} 121 | 122 | 123 | sign-artifacts 124 | verify 125 | 126 | sign 127 | 128 | 129 | 130 | 131 | 132 | org.sonatype.plugins 133 | nexus-staging-maven-plugin 134 | ${nexus.staging.maven.plugin.version} 135 | true 136 | 137 | ossrh 138 | https://oss.sonatype.org/ 139 | 140 | false 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | -------------------------------------------------------------------------------- /cassandra-2/src/main/java/com/instaclustr/cassandra/ttl/NoTTLSSTableIdentityIterator.java: -------------------------------------------------------------------------------- 1 | package com.instaclustr.cassandra.ttl; 2 | 3 | import java.io.DataInput; 4 | import java.io.IOError; 5 | import java.io.IOException; 6 | import java.util.Iterator; 7 | 8 | import org.apache.cassandra.config.CFMetaData; 9 | import org.apache.cassandra.config.DatabaseDescriptor; 10 | import org.apache.cassandra.db.ArrayBackedSortedColumns; 11 | import org.apache.cassandra.db.ColumnFamily; 12 | import org.apache.cassandra.db.ColumnSerializer; 13 | import org.apache.cassandra.db.DecoratedKey; 14 | import org.apache.cassandra.db.DeletionTime; 15 | import org.apache.cassandra.db.OnDiskAtom; 16 | import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; 17 | import org.apache.cassandra.io.sstable.CorruptSSTableException; 18 | import org.apache.cassandra.io.util.FileDataInput; 19 | import org.apache.cassandra.io.util.RandomAccessReader; 20 | import org.apache.cassandra.serializers.MarshalException; 21 | 22 | /** 23 | * Created by Linda on 8/05/2016. 24 | */ 25 | public class NoTTLSSTableIdentityIterator implements Comparable, OnDiskAtomIterator { 26 | 27 | private final DecoratedKey key; 28 | public final ColumnSerializer.Flag flag; 29 | 30 | private final ColumnFamily columnFamily; 31 | private final Iterator atomIterator; 32 | private final boolean validateColumns; 33 | private final String filename; 34 | 35 | private final NoTTLReader sstable; 36 | 37 | public NoTTLSSTableIdentityIterator(NoTTLReader sstable, RandomAccessReader file, DecoratedKey key) { 38 | this(sstable, file, key, false); 39 | } 40 | 41 | public NoTTLSSTableIdentityIterator(NoTTLReader sstable, RandomAccessReader file, DecoratedKey key, boolean checkData) { 42 | this(sstable.metadata, file, file.getPath(), key, checkData, sstable, ColumnSerializer.Flag.LOCAL); 43 | } 44 | 45 | 46 | private NoTTLSSTableIdentityIterator(CFMetaData metadata, 47 | FileDataInput in, 48 | String filename, 49 | DecoratedKey key, 50 | boolean checkData, //false 51 | NoTTLReader sstable, 52 | ColumnSerializer.Flag flag) //local 53 | { 54 | this(metadata, in, filename, key, checkData, sstable, flag, readDeletionTime(in, sstable, filename), 55 | NoTTLAbstractCell.onDiskIterator(in, flag, (int) (System.currentTimeMillis() / 1000), 56 | sstable == null ? DatabaseDescriptor.getSSTableFormat().info.getLatestVersion() : sstable.descriptor.version, metadata.comparator)); 57 | } 58 | 59 | private NoTTLSSTableIdentityIterator(CFMetaData metadata, 60 | DataInput in, 61 | String filename, 62 | DecoratedKey key, 63 | boolean checkData, 64 | NoTTLReader sstable, 65 | ColumnSerializer.Flag flag, 66 | DeletionTime deletion, 67 | Iterator atomIterator) { 68 | assert !checkData || (sstable != null); 69 | this.filename = filename; 70 | this.key = key; 71 | this.flag = flag; 72 | this.validateColumns = checkData; 73 | this.sstable = sstable; 74 | columnFamily = ArrayBackedSortedColumns.factory.create(metadata); 75 | columnFamily.delete(deletion); 76 | this.atomIterator = atomIterator; 77 | } 78 | 79 | private static DeletionTime readDeletionTime(DataInput in, NoTTLReader sstable, String filename) { 80 | try { 81 | return DeletionTime.serializer.deserialize(in); 82 | } catch (IOException e) { 83 | if (sstable != null) { 84 | sstable.markSuspect(); 85 | } 86 | throw new CorruptSSTableException(e, filename); 87 | } 88 | } 89 | 90 | 91 | public ColumnFamily getColumnFamily() { 92 | return columnFamily; 93 | } 94 | 95 | public DecoratedKey getKey() { 96 | return key; 97 | } 98 | 99 | public void close() throws IOException { 100 | 101 | } 102 | 103 | public boolean hasNext() { 104 | try { 105 | return atomIterator.hasNext(); 106 | } catch (IOError e) { 107 | // catch here b/c atomIterator is an AbstractIterator; hasNext reads the value 108 | if (e.getCause() instanceof IOException) { 109 | if (sstable != null) { 110 | sstable.markSuspect(); 111 | } 112 | throw new CorruptSSTableException((IOException) e.getCause(), filename); 113 | } else { 114 | throw e; 115 | } 116 | } 117 | } 118 | 119 | public OnDiskAtom next() { 120 | try { 121 | OnDiskAtom atom = atomIterator.next(); 122 | if (validateColumns) { 123 | atom.validateFields(columnFamily.metadata()); 124 | } 125 | return atom; 126 | } catch (MarshalException me) { 127 | throw new CorruptSSTableException(me, filename); 128 | } 129 | } 130 | 131 | public int compareTo(NoTTLSSTableIdentityIterator o) { 132 | return key.compareTo(o.key); 133 | } 134 | } -------------------------------------------------------------------------------- /cassandra-2/src/main/java/com/instaclustr/cassandra/ttl/Cassandra2TTLRemover.java: -------------------------------------------------------------------------------- 1 | package com.instaclustr.cassandra.ttl; 2 | 3 | import static java.lang.String.format; 4 | 5 | import java.nio.file.Path; 6 | import java.util.Collection; 7 | import java.util.Iterator; 8 | 9 | import com.instaclustr.cassandra.ttl.cli.TTLRemovalException; 10 | import org.apache.cassandra.config.Config; 11 | import org.apache.cassandra.config.DatabaseDescriptor; 12 | import org.apache.cassandra.config.Schema; 13 | import org.apache.cassandra.db.ArrayBackedSortedColumns; 14 | import org.apache.cassandra.db.BufferDeletedCell; 15 | import org.apache.cassandra.db.BufferExpiringCell; 16 | import org.apache.cassandra.db.Cell; 17 | import org.apache.cassandra.db.ColumnFamily; 18 | import org.apache.cassandra.db.Keyspace; 19 | import org.apache.cassandra.db.OnDiskAtom; 20 | import org.apache.cassandra.io.sstable.Descriptor; 21 | import org.apache.cassandra.io.sstable.Descriptor.Type; 22 | import org.apache.cassandra.io.sstable.ISSTableScanner; 23 | import org.apache.cassandra.io.sstable.KeyIterator; 24 | import org.apache.cassandra.io.sstable.format.SSTableFormat; 25 | import org.apache.cassandra.io.sstable.format.SSTableWriter; 26 | import org.apache.cassandra.service.ActiveRepairService; 27 | import org.slf4j.Logger; 28 | import org.slf4j.LoggerFactory; 29 | 30 | public class Cassandra2TTLRemover implements SSTableTTLRemover { 31 | 32 | private static final Logger logger = LoggerFactory.getLogger(Cassandra2TTLRemover.class); 33 | 34 | @Override 35 | public void executeRemoval(final Path outputFolder, final Collection sstables, final String cql) throws Exception { 36 | 37 | if (!Boolean.parseBoolean(System.getProperty("ttl.remover.tests", "false"))) { 38 | DatabaseDescriptor.forceStaticInitialization(); 39 | } 40 | 41 | Config.setClientMode(false); 42 | 43 | if (Boolean.parseBoolean(System.getProperty("ttl.remover.tests", "false"))) { 44 | DatabaseDescriptor.applyConfig(DatabaseDescriptor.loadConfig()); 45 | } 46 | 47 | try { 48 | Schema.instance.loadFromDisk(false); 49 | } catch (final Exception ex) { 50 | // important as that would fail when we are on tests 51 | } 52 | Keyspace.setInitialized(); 53 | 54 | for (final Path sstable : sstables) { 55 | 56 | final Descriptor descriptor = Descriptor.fromFilename(sstable.toAbsolutePath().toFile().getAbsolutePath()); 57 | 58 | if (Schema.instance.getKSMetaData(descriptor.ksname) == null) { 59 | logger.warn(format("Filename %s references to nonexistent keyspace: %s!", sstable, descriptor.ksname)); 60 | continue; 61 | } 62 | 63 | logger.info(format("Loading file %s from initial keyspace: %s", sstable, descriptor.ksname)); 64 | 65 | final Path newSSTableDestinationDir = outputFolder.resolve(descriptor.ksname).resolve(descriptor.cfname); 66 | 67 | if (!newSSTableDestinationDir.toFile().exists()) { 68 | if (!newSSTableDestinationDir.toFile().mkdirs()) { 69 | throw new TTLRemovalException(format("Unable to create directories leading to %s.", newSSTableDestinationDir.toFile().getAbsolutePath())); 70 | } 71 | } 72 | 73 | final Descriptor resultDesc = new Descriptor(newSSTableDestinationDir.toFile(), 74 | descriptor.ksname, 75 | descriptor.cfname, 76 | descriptor.generation, 77 | Type.FINAL, 78 | SSTableFormat.Type.BIG); 79 | 80 | stream(descriptor, resultDesc); 81 | } 82 | } 83 | 84 | public void stream(final Descriptor descriptor, final Descriptor toSSTable) throws TTLRemovalException { 85 | 86 | ISSTableScanner noTTLscanner = null; 87 | 88 | try { 89 | long keyCount = countKeys(descriptor); 90 | 91 | NoTTLReader noTTLreader = NoTTLReader.open(descriptor); 92 | 93 | noTTLscanner = noTTLreader.getScanner(); 94 | 95 | ColumnFamily columnFamily = ArrayBackedSortedColumns.factory.create(descriptor.ksname, descriptor.cfname); 96 | 97 | SSTableWriter writer = SSTableWriter.create(toSSTable, keyCount, ActiveRepairService.UNREPAIRED_SSTABLE); 98 | 99 | NoTTLSSTableIdentityIterator row; 100 | 101 | while (noTTLscanner.hasNext()) //read data from disk //NoTTLBigTableScanner 102 | { 103 | row = (NoTTLSSTableIdentityIterator) noTTLscanner.next(); 104 | serializeRow(row, columnFamily); 105 | writer.append(row.getKey(), columnFamily); 106 | columnFamily.clear(); 107 | } 108 | 109 | writer.finish(true); 110 | } catch (final Exception ex) { 111 | throw new TTLRemovalException("Unable to remove TTL from sstables.", ex); 112 | } finally { 113 | if (noTTLscanner != null) { 114 | try { 115 | noTTLscanner.close(); 116 | } catch (final Exception ex) { 117 | throw new TTLRemovalException("Unable to close TTL scanner", ex); 118 | } 119 | } 120 | } 121 | } 122 | 123 | private void serializeRow(Iterator atoms, ColumnFamily columnFamily) { 124 | 125 | while (atoms.hasNext()) { 126 | serializeAtom(atoms.next(), columnFamily); 127 | } 128 | 129 | } 130 | 131 | private void serializeAtom(OnDiskAtom atom, ColumnFamily columnFamily) { 132 | if (atom instanceof Cell) { 133 | Cell cell = (Cell) atom; 134 | if (cell instanceof BufferExpiringCell) { 135 | columnFamily.addColumn(cell.name(), cell.value(), cell.timestamp()); 136 | } else if (cell instanceof BufferDeletedCell) { 137 | columnFamily.addColumn(cell); 138 | } else { 139 | columnFamily.addColumn(cell); 140 | } 141 | 142 | } 143 | } 144 | 145 | private long countKeys(Descriptor descriptor) { 146 | KeyIterator iter = new KeyIterator(descriptor); 147 | long keycount = 0; 148 | try { 149 | while (iter.hasNext()) { 150 | iter.next(); 151 | keycount++; 152 | } 153 | 154 | } finally { 155 | iter.close(); 156 | } 157 | 158 | return keycount; 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /mvnw: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # ---------------------------------------------------------------------------- 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # ---------------------------------------------------------------------------- 20 | 21 | # ---------------------------------------------------------------------------- 22 | # Maven2 Start Up Batch script 23 | # 24 | # Required ENV vars: 25 | # ------------------ 26 | # JAVA_HOME - location of a JDK home dir 27 | # 28 | # Optional ENV vars 29 | # ----------------- 30 | # M2_HOME - location of maven2's installed home dir 31 | # MAVEN_OPTS - parameters passed to the Java VM when running Maven 32 | # e.g. to debug Maven itself, use 33 | # set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 34 | # MAVEN_SKIP_RC - flag to disable loading of mavenrc files 35 | # ---------------------------------------------------------------------------- 36 | 37 | if [ -z "$MAVEN_SKIP_RC" ] ; then 38 | 39 | if [ -f /etc/mavenrc ] ; then 40 | . /etc/mavenrc 41 | fi 42 | 43 | if [ -f "$HOME/.mavenrc" ] ; then 44 | . "$HOME/.mavenrc" 45 | fi 46 | 47 | fi 48 | 49 | # OS specific support. $var _must_ be set to either true or false. 50 | cygwin=false; 51 | darwin=false; 52 | mingw=false 53 | case "`uname`" in 54 | CYGWIN*) cygwin=true ;; 55 | MINGW*) mingw=true;; 56 | Darwin*) darwin=true 57 | # 58 | # Look for the Apple JDKs first to preserve the existing behaviour, and then look 59 | # for the new JDKs provided by Oracle. 60 | # 61 | if [ -z "$JAVA_HOME" ] && [ -L /System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK ] ; then 62 | # 63 | # Apple JDKs 64 | # 65 | export JAVA_HOME=/System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK/Home 66 | fi 67 | 68 | if [ -z "$JAVA_HOME" ] && [ -L /System/Library/Java/JavaVirtualMachines/CurrentJDK ] ; then 69 | # 70 | # Apple JDKs 71 | # 72 | export JAVA_HOME=/System/Library/Java/JavaVirtualMachines/CurrentJDK/Contents/Home 73 | fi 74 | 75 | if [ -z "$JAVA_HOME" ] && [ -L "/Library/Java/JavaVirtualMachines/CurrentJDK" ] ; then 76 | # 77 | # Oracle JDKs 78 | # 79 | export JAVA_HOME=/Library/Java/JavaVirtualMachines/CurrentJDK/Contents/Home 80 | fi 81 | 82 | if [ -z "$JAVA_HOME" ] && [ -x "/usr/libexec/java_home" ]; then 83 | # 84 | # Apple JDKs 85 | # 86 | export JAVA_HOME=`/usr/libexec/java_home` 87 | fi 88 | ;; 89 | esac 90 | 91 | if [ -z "$JAVA_HOME" ] ; then 92 | if [ -r /etc/gentoo-release ] ; then 93 | JAVA_HOME=`java-config --jre-home` 94 | fi 95 | fi 96 | 97 | if [ -z "$M2_HOME" ] ; then 98 | ## resolve links - $0 may be a link to maven's home 99 | PRG="$0" 100 | 101 | # need this for relative symlinks 102 | while [ -h "$PRG" ] ; do 103 | ls=`ls -ld "$PRG"` 104 | link=`expr "$ls" : '.*-> \(.*\)$'` 105 | if expr "$link" : '/.*' > /dev/null; then 106 | PRG="$link" 107 | else 108 | PRG="`dirname "$PRG"`/$link" 109 | fi 110 | done 111 | 112 | saveddir=`pwd` 113 | 114 | M2_HOME=`dirname "$PRG"`/.. 115 | 116 | # make it fully qualified 117 | M2_HOME=`cd "$M2_HOME" && pwd` 118 | 119 | cd "$saveddir" 120 | # echo Using m2 at $M2_HOME 121 | fi 122 | 123 | # For Cygwin, ensure paths are in UNIX format before anything is touched 124 | if $cygwin ; then 125 | [ -n "$M2_HOME" ] && 126 | M2_HOME=`cygpath --unix "$M2_HOME"` 127 | [ -n "$JAVA_HOME" ] && 128 | JAVA_HOME=`cygpath --unix "$JAVA_HOME"` 129 | [ -n "$CLASSPATH" ] && 130 | CLASSPATH=`cygpath --path --unix "$CLASSPATH"` 131 | fi 132 | 133 | # For Migwn, ensure paths are in UNIX format before anything is touched 134 | if $mingw ; then 135 | [ -n "$M2_HOME" ] && 136 | M2_HOME="`(cd "$M2_HOME"; pwd)`" 137 | [ -n "$JAVA_HOME" ] && 138 | JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" 139 | # TODO classpath? 140 | fi 141 | 142 | if [ -z "$JAVA_HOME" ]; then 143 | javaExecutable="`which javac`" 144 | if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then 145 | # readlink(1) is not available as standard on Solaris 10. 146 | readLink=`which readlink` 147 | if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then 148 | if $darwin ; then 149 | javaHome="`dirname \"$javaExecutable\"`" 150 | javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" 151 | else 152 | javaExecutable="`readlink -f \"$javaExecutable\"`" 153 | fi 154 | javaHome="`dirname \"$javaExecutable\"`" 155 | javaHome=`expr "$javaHome" : '\(.*\)/bin'` 156 | JAVA_HOME="$javaHome" 157 | export JAVA_HOME 158 | fi 159 | fi 160 | fi 161 | 162 | if [ -z "$JAVACMD" ] ; then 163 | if [ -n "$JAVA_HOME" ] ; then 164 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 165 | # IBM's JDK on AIX uses strange locations for the executables 166 | JAVACMD="$JAVA_HOME/jre/sh/java" 167 | else 168 | JAVACMD="$JAVA_HOME/bin/java" 169 | fi 170 | else 171 | JAVACMD="`which java`" 172 | fi 173 | fi 174 | 175 | if [ ! -x "$JAVACMD" ] ; then 176 | echo "Error: JAVA_HOME is not defined correctly." >&2 177 | echo " We cannot execute $JAVACMD" >&2 178 | exit 1 179 | fi 180 | 181 | if [ -z "$JAVA_HOME" ] ; then 182 | echo "Warning: JAVA_HOME environment variable is not set." 183 | fi 184 | 185 | CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher 186 | 187 | # For Cygwin, switch paths to Windows format before running java 188 | if $cygwin; then 189 | [ -n "$M2_HOME" ] && 190 | M2_HOME=`cygpath --path --windows "$M2_HOME"` 191 | [ -n "$JAVA_HOME" ] && 192 | JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` 193 | [ -n "$CLASSPATH" ] && 194 | CLASSPATH=`cygpath --path --windows "$CLASSPATH"` 195 | fi 196 | 197 | # traverses directory structure from process work directory to filesystem root 198 | # first directory with .mvn subdirectory is considered project base directory 199 | find_maven_basedir() { 200 | local basedir=$(pwd) 201 | local wdir=$(pwd) 202 | while [ "$wdir" != '/' ] ; do 203 | if [ -d "$wdir"/.mvn ] ; then 204 | basedir=$wdir 205 | break 206 | fi 207 | wdir=$(cd "$wdir/.."; pwd) 208 | done 209 | echo "${basedir}" 210 | } 211 | 212 | # concatenates all lines of a file 213 | concat_lines() { 214 | if [ -f "$1" ]; then 215 | echo "$(tr -s '\n' ' ' < "$1")" 216 | fi 217 | } 218 | 219 | export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-$(find_maven_basedir)} 220 | MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" 221 | 222 | # Provide a "standardized" way to retrieve the CLI args that will 223 | # work with both Windows and non-Windows executions. 224 | MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@" 225 | export MAVEN_CMD_LINE_ARGS 226 | 227 | WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 228 | 229 | exec "$JAVACMD" \ 230 | $MAVEN_OPTS \ 231 | -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ 232 | "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ 233 | ${WRAPPER_LAUNCHER} $MAVEN_CMD_LINE_ARGS 234 | 235 | -------------------------------------------------------------------------------- /README.adoc: -------------------------------------------------------------------------------- 1 | # Cassandra TTL Remover 2 | 3 | _Tool for rewriting SSTables to remove TTLs_ 4 | 5 | image:https://img.shields.io/maven-central/v/com.instaclustr/ttl-remover-impl.svg?label=Maven%20Central[link=https://search.maven.org/search?q=g:%22com.instaclustr%22%20AND%20a:%22ttl-remover-impl%22] 6 | image:https://circleci.com/gh/instaclustr/cassandra-ttl-remover.svg?style=svg["Instaclustr",link="https://circleci.com/gh/instaclustr/cassandra-ttl-remover"] 7 | 8 | - Website: https://www.instaclustr.com/ 9 | - Documentation: https://www.instaclustr.com/support/documentation/ 10 | 11 | TTL remover removes TTL information for SSTables by rewriting them and creating new ones, so it looks like data has never expired and they will never expire either. 12 | This is handy for testing scenarios and debugging purposes when we want to have data in Cassandra, visible, but the underlying SSTable has expired them in the meanwhile. 13 | You can then load them back e.g. by `sstableloader` to Cassandra. 14 | 15 | We are supporting: 16 | 17 | * Cassandra 2 line (2.2.19) 18 | * Cassandra 3 line (3.11.14) 19 | * Cassandra 4 line (4.0.7) 20 | * Cassandra 4.1 line (4.1.0) 21 | 22 | ### Usage 23 | 24 | The user of this software might either grab the binaries in Maven Central, or they may build it on their own. 25 | 26 | The project consists of these modules: 27 | 28 | * impl - the implementation of CLI plugin 29 | * cassandra-{2,3,4} - the implementation of TTL remover 30 | * buddy-agent - Byte Buddy agent used upon Cassandra 3 and 4 TTL removal 31 | 32 | Each remover for each respective Cassandra version removes TTLs from SSTables a little bit differently. 33 | It is notoriously known that Cassandra is little bit hairy when it comes to re-usability in other projects, 34 | so we are extending the modifying of some Cassandra classes by copying them over and rewriting stuff which 35 | can not be out of the box (e.g. the case for Cassandra 2.2.x is particularly strong here). 36 | 37 | The `impl` module contains an interface `SSTableTTLRemover` which all Cassandra-specific modules 38 | implement. The SPI mechanism will load the concrete remover just because it was put on a class path. 39 | Hence, the mechanism to switch between Cassandra versions is to place the correct implementation 40 | JAR on the class path and the `impl` module will do the rest. 41 | 42 | `buddy-agent` contains an agent which is used upon execution for Cassandra 3 and 4 TTL removal. The purpose of this 43 | module is to mock some `DatabaseDescriptor` static methods. Normally, this class is initialized when a Cassandra process is run, 44 | but we are not running anything. The removal logic uses these methods—normally we would have to have 45 | proper database schema in `$CASSANDRA_HOME/data` and logic which deals with reading and writing SSTables, and for example 46 | would also fetch data from system tables. This is not desirable and by introducing this module 47 | the only thing necessary is `$CASSANDRA_HOME` with libs/jars so we can populate the classpath. 48 | 49 | The released artifacts do not ship Cassandra with it—you have to have `$CASSANDRA_HOME` set—pointing 50 | to your Cassandra installation from which you need to remove TTL information from SSTables. 51 | 52 | ### run.sh script 53 | 54 | It is recommended to use `run.sh` helper script if you want to remove TTLs. You are welcome to 55 | modify this script at its end to support your case. At the bottom, you see: 56 | 57 | ---- 58 | CLASSPATH=$CLASSPATH./impl/target/ttl-remover.jar:./cassandra-2/target/ttl-remover-cassandra-2.jar 59 | 60 | java -cp "$CLASSPATH./impl/target/ttl-remover.jar:./cassandra-2/target/ttl-remover-cassandra-2.jar" 61 | \$JVM_OPTS \ 62 | com.instaclustr.cassandra.ttl.cli.TTLRemoverCLI \ 63 | --cassandra-version=2 \ 64 | --sstables \ 65 | /tmp/sstables2/test \ 66 | --output-path \ 67 | /tmp/stripped \ 68 | --cassandra-yaml \ 69 | $CASSANDRA_HOME/conf/cassandra.yaml \ 70 | --cassandra-storage-dir \ 71 | $CASSANDRA_HOME/data 72 | ---- 73 | 74 | On the other hand, for Cassandra 3/4, the command would look like this. Notice we are using 75 | byte-buddy agent, unlike for Cassandra 2 case, and we are also specifying CQL statement as we 76 | not have have access to `data` dir (it may be completely empty) but we need to construct metadata 77 | upon TTL removal, so we do it programmatically. 78 | 79 | ---- 80 | java -javaagent:./buddy-agent/target/byte-buddy-agent.jar \ 81 | -cp "$CLASSPATH./impl/target/ttl-remover.jar:./cassandra-3/target/ttl-remover-cassandra-3.jar" \ 82 | $JVM_OPTS \ 83 | com.instaclustr.cassandra.ttl.cli.TTLRemoverCLI \ 84 | --cassandra-version=3 \ 85 | --sstables \ 86 | /tmp/original-3/test/test \ 87 | --output-path \ 88 | /tmp/stripped \ 89 | --cql \ 90 | 'CREATE TABLE IF NOT EXISTS test.test (id uuid, name text, surname text, PRIMARY KEY (id)) WITH default_time_to_live = 10;' 91 | ---- 92 | 93 | All configuration options are as follows—you get this by the `help` command just after class specification: 94 | 95 | ---- 96 | Usage: ttl-remove [-hV] [-c=[INTEGER]] [-d=[DIRECTORY]] [-f=[FILE]] -p= 97 | [DIRECTORY] [-q=[CQL]] [-s=[DIRECTORY]] [-t=[FILE]] 98 | command for removing TTL from SSTables 99 | -p, --output-path=[DIRECTORY] 100 | Destination where SSTable will be generated. 101 | -f, --cassandra-yaml=[FILE] 102 | Path to cassandra.yaml file for loading generated 103 | SSTables to Cassandra, relevant only in case of 104 | Cassandra 2. 105 | -d, --cassandra-storage-dir=[DIRECTORY] 106 | Path to cassandra data dir, relevant only in case of 107 | Cassandra 2. 108 | -s, --sstables=[DIRECTORY] 109 | Path to a directory for which all SSTables will have 110 | TTL removed. 111 | -t, --sstable=[FILE] Path to .db file of a SSTable which will have TTL 112 | removed. 113 | -c, --cassandra-version=[INTEGER] 114 | Version of Cassandra to remove TTL for, might be 2, 3 115 | or 4, defaults to 3 116 | -q, --cql=[CQL] CQL statement which creates table we want to remove 117 | TTL from. This has to be set in case 118 | --cassandra-version is 3 or 4 119 | -h, --help Show this help message and exit. 120 | -V, --version Print version information and exit. 121 | 122 | ---- 123 | 124 | `--cassandra-storage-dir` is the directory where all your data/SSTables are 125 | for your Cassandra installation. This has to be specified; normally it would point to something like 126 | `/var/lib/cassandra/data` to give an example. This has to be specified explicitly. 127 | 128 | `--cassandra-storage-dir` and `--cassandra-yaml` are only necessary upon Cassandra 2 TTL removal. 129 | 130 | `--cassandra-yaml` is the path to your `cassandra.yaml`; this has to be specified explicitly too. 131 | 132 | Lastly, there has to be `--output-path` specified too—where your stripped SSTables from TTLs should be. 133 | 134 | ### Load TTL-Removed SSTable to a New Cluster 135 | 136 | 1. Create the keyspace and table of the target SStable in the new cluster. 137 | 138 | 2. In the source cluster, use the following command to load the ttl-removed SSTable into the new cluster. 139 | 140 | ./sstableloader -d [path to the ttl-removed sstable folder] 141 | 142 | ### Build 143 | 144 | ---- 145 | $ mvn clean install 146 | ---- 147 | 148 | Tests are skipped by `mvn clean install -DskipTests`. 149 | 150 | Please be sure that your $CASSANDRA_HOME **is not** set. Unit tests are starting an embedded Cassandra 151 | instance which is setting its own "Cassandra home", and having this set externally would confuse tests 152 | as it would react to a different Cassandra home. 153 | 154 | ### Further Information 155 | 156 | See Danyang Li's blog ["TTLRemover: Tool for Removing Cassandra TTLs for Recovery and Testing Purposes"](https://www.instaclustr.com/ttlremover-tool-for-removing-cassandra-ttls-for-recovery-and-testing-purposes/) 157 | 158 | Please see https://www.instaclustr.com/support/documentation/announcements/instaclustr-open-source-project-status/ for Instaclustr support status of this project 159 | -------------------------------------------------------------------------------- /cassandra-4/src/main/java/com/instaclustr/cassandra/ttl/Cassandra4TTLRemover.java: -------------------------------------------------------------------------------- 1 | package com.instaclustr.cassandra.ttl; 2 | 3 | import static java.lang.String.format; 4 | 5 | import java.nio.file.Path; 6 | import java.util.Collection; 7 | import java.util.Collections; 8 | 9 | import com.instaclustr.cassandra.ttl.cli.TTLRemovalException; 10 | import org.apache.cassandra.cql3.statements.schema.CreateTableStatement; 11 | import org.apache.cassandra.db.ClusteringBound; 12 | import org.apache.cassandra.db.LivenessInfo; 13 | import org.apache.cassandra.db.RangeTombstone; 14 | import org.apache.cassandra.db.SerializationHeader; 15 | import org.apache.cassandra.db.Slice; 16 | import org.apache.cassandra.db.compaction.OperationType; 17 | import org.apache.cassandra.db.lifecycle.LifecycleTransaction; 18 | import org.apache.cassandra.db.partitions.PartitionUpdate; 19 | import org.apache.cassandra.db.rows.*; 20 | import org.apache.cassandra.db.rows.Row.Builder; 21 | import org.apache.cassandra.dht.Murmur3Partitioner; 22 | import org.apache.cassandra.io.sstable.Descriptor; 23 | import org.apache.cassandra.io.sstable.ISSTableScanner; 24 | import org.apache.cassandra.io.sstable.KeyIterator; 25 | import org.apache.cassandra.io.sstable.SSTable; 26 | import org.apache.cassandra.io.sstable.SSTableRewriter; 27 | import org.apache.cassandra.io.sstable.format.SSTableFormat; 28 | import org.apache.cassandra.io.sstable.format.SSTableReader; 29 | import org.apache.cassandra.io.sstable.format.SSTableWriter; 30 | import org.apache.cassandra.schema.ColumnMetadata; 31 | import org.apache.cassandra.schema.TableMetadata; 32 | import org.apache.cassandra.schema.TableMetadataRef; 33 | import org.apache.cassandra.utils.FBUtilities; 34 | import org.slf4j.Logger; 35 | import org.slf4j.LoggerFactory; 36 | 37 | public class Cassandra4TTLRemover implements SSTableTTLRemover { 38 | 39 | private static final Logger logger = LoggerFactory.getLogger(Cassandra4TTLRemover.class); 40 | 41 | @Override 42 | public void executeRemoval(final Path outputFolder, final Collection sstables, final String cql) throws Exception { 43 | 44 | for (final Path sstable : sstables) { 45 | final Descriptor descriptor = Descriptor.fromFilename(sstable.toAbsolutePath().toFile().getAbsolutePath()); 46 | 47 | logger.info(format("Loading file %s from initial keyspace: %s", sstable, descriptor.ksname)); 48 | 49 | final Path newSSTableDestinationDir = outputFolder.resolve(descriptor.ksname).resolve(descriptor.cfname); 50 | 51 | if (!newSSTableDestinationDir.toFile().exists()) { 52 | if (!newSSTableDestinationDir.toFile().mkdirs()) { 53 | throw new TTLRemovalException(format("Unable to create directories leading to %s.", newSSTableDestinationDir.toFile().getAbsolutePath())); 54 | } 55 | } 56 | 57 | final Descriptor resultDesc = new Descriptor(newSSTableDestinationDir.toFile(), 58 | descriptor.ksname, 59 | descriptor.cfname, 60 | descriptor.generation, 61 | SSTableFormat.Type.BIG); 62 | 63 | final TableMetadata tableMetadata = CreateTableStatement.parse(cql, descriptor.ksname).partitioner(new Murmur3Partitioner()).build(); 64 | 65 | stream(descriptor, resultDesc, tableMetadata); 66 | } 67 | } 68 | 69 | public void stream(final Descriptor descriptor, final Descriptor toSSTable, final TableMetadata tableMetadata) throws TTLRemovalException { 70 | 71 | final SSTableReader noTTLReader; 72 | 73 | try { 74 | noTTLReader = SSTableReader.open(descriptor, SSTable.componentsFor(descriptor), TableMetadataRef.forOfflineTools(tableMetadata), true, true); 75 | } catch (final Exception ex) { 76 | throw new TTLRemovalException(format("Unable to open descriptor %s", descriptor.baseFilename()), ex); 77 | } 78 | 79 | final long keyCount = countKeys(descriptor, tableMetadata); 80 | 81 | final LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE); 82 | 83 | final SerializationHeader header = SerializationHeader.make(tableMetadata, Collections.singletonList(noTTLReader)); 84 | 85 | final SSTableRewriter writer = SSTableRewriter.constructKeepingOriginals(txn, true, Long.MAX_VALUE); 86 | 87 | writer.switchWriter(SSTableWriter.create(TableMetadataRef.forOfflineTools(tableMetadata), toSSTable, keyCount, -1, null, false, 0, header, null, txn)); 88 | 89 | try (final ISSTableScanner sourceSSTableScanner = noTTLReader.getScanner()) { 90 | while (sourceSSTableScanner.hasNext()) { 91 | final UnfilteredRowIterator partition = sourceSSTableScanner.next(); 92 | 93 | if (!partition.hasNext()) { 94 | //keep partitions with no rows 95 | writer.append(partition); 96 | continue; 97 | } 98 | 99 | final PartitionUpdate.Builder builder = new PartitionUpdate.Builder(tableMetadata, 100 | partition.partitionKey(), 101 | partition.columns(), 102 | 2, 103 | false); 104 | 105 | while (partition.hasNext()) { 106 | 107 | final Unfiltered unfiltered = partition.next(); 108 | 109 | switch (unfiltered.kind()) { 110 | case ROW: 111 | Row newRow = serializeRow(unfiltered); 112 | builder.add(newRow); 113 | break; 114 | case RANGE_TOMBSTONE_MARKER: 115 | //Range tombstones are denoted as separate (Unfiltered) entries for start and end, 116 | //so we record them separately and add the tombstone once both ends of the range are defined 117 | RangeTombstoneBoundMarker marker = (RangeTombstoneBoundMarker) unfiltered; 118 | 119 | final ClusteringBound start = marker.isOpen(false) ? marker.openBound(false) : null; 120 | final ClusteringBound end = marker.isClose(false) ? marker.closeBound(false) : null; 121 | 122 | if (start != null && end != null) { 123 | builder.add(new RangeTombstone(Slice.make(start, end), marker.deletionTime())); 124 | } 125 | 126 | break; 127 | } 128 | } 129 | 130 | writer.append(builder.build().unfilteredIterator()); 131 | } 132 | writer.finish(); 133 | } catch (final Exception ex) { 134 | throw new TTLRemovalException(format("Exception occurred while scanning SSTable %s", descriptor.baseFilename()), ex); 135 | } 136 | } 137 | 138 | private long countKeys(final Descriptor descriptor, final TableMetadata metaData) { 139 | 140 | final KeyIterator iter = new KeyIterator(descriptor, metaData); 141 | 142 | long keyCount = 0; 143 | 144 | try { 145 | while (iter.hasNext()) { 146 | iter.next(); 147 | keyCount++; 148 | } 149 | } finally { 150 | iter.close(); 151 | } 152 | 153 | return keyCount; 154 | } 155 | 156 | private Row serializeRow(final Unfiltered atoms) { 157 | 158 | final Row row = (Row) atoms; 159 | 160 | Builder builder = BTreeRow.sortedBuilder(); 161 | 162 | builder.newRow(row.clustering()); 163 | builder.addPrimaryKeyLivenessInfo(LivenessInfo.create(row.primaryKeyLivenessInfo().timestamp(), 164 | LivenessInfo.NO_TTL, 165 | FBUtilities.nowInSeconds())); 166 | 167 | row.columnData().forEach(cd -> { 168 | ColumnMetadata columnMetadata = cd.column(); 169 | if (columnMetadata.isComplex()) { 170 | for (Cell cell : row.getComplexColumnData(columnMetadata)) { 171 | builder.addCell(BufferCell.live(cell.column(), cell.timestamp(), cell.buffer(), cell.path())); 172 | } 173 | } else { 174 | Cell cell = row.getCell(columnMetadata); 175 | builder.addCell(BufferCell.live(cell.column(), cell.timestamp(), cell.buffer())); 176 | } 177 | }); 178 | builder.addRowDeletion(row.deletion()); 179 | 180 | return builder.build(); 181 | } 182 | } 183 | -------------------------------------------------------------------------------- /cassandra-4.1/src/main/java/com/instaclustr/cassandra/ttl/Cassandra41TTLRemover.java: -------------------------------------------------------------------------------- 1 | package com.instaclustr.cassandra.ttl; 2 | 3 | import static java.lang.String.format; 4 | 5 | import java.nio.file.Path; 6 | import java.util.Collection; 7 | import java.util.Collections; 8 | 9 | import com.instaclustr.cassandra.ttl.cli.TTLRemovalException; 10 | import org.apache.cassandra.config.DatabaseDescriptor; 11 | import org.apache.cassandra.cql3.statements.schema.CreateTableStatement; 12 | import org.apache.cassandra.db.ClusteringBound; 13 | import org.apache.cassandra.db.LivenessInfo; 14 | import org.apache.cassandra.db.RangeTombstone; 15 | import org.apache.cassandra.db.SerializationHeader; 16 | import org.apache.cassandra.db.Slice; 17 | import org.apache.cassandra.db.compaction.OperationType; 18 | import org.apache.cassandra.db.lifecycle.LifecycleTransaction; 19 | import org.apache.cassandra.db.partitions.PartitionUpdate; 20 | import org.apache.cassandra.db.rows.*; 21 | import org.apache.cassandra.db.rows.Row.Builder; 22 | import org.apache.cassandra.dht.Murmur3Partitioner; 23 | import org.apache.cassandra.io.sstable.Descriptor; 24 | import org.apache.cassandra.io.sstable.ISSTableScanner; 25 | import org.apache.cassandra.io.sstable.KeyIterator; 26 | import org.apache.cassandra.io.sstable.SSTable; 27 | import org.apache.cassandra.io.sstable.SSTableRewriter; 28 | import org.apache.cassandra.io.sstable.format.SSTableFormat; 29 | import org.apache.cassandra.io.sstable.format.SSTableReader; 30 | import org.apache.cassandra.io.sstable.format.SSTableWriter; 31 | import org.apache.cassandra.io.util.File; 32 | import org.apache.cassandra.schema.ColumnMetadata; 33 | import org.apache.cassandra.schema.TableMetadata; 34 | import org.apache.cassandra.schema.TableMetadataRef; 35 | import org.apache.cassandra.utils.FBUtilities; 36 | import org.slf4j.Logger; 37 | import org.slf4j.LoggerFactory; 38 | 39 | public class Cassandra41TTLRemover implements SSTableTTLRemover { 40 | 41 | private static final Logger logger = LoggerFactory.getLogger(Cassandra41TTLRemover.class); 42 | 43 | @Override 44 | public void executeRemoval(final Path outputFolder, final Collection sstables, final String cql) throws Exception { 45 | DatabaseDescriptor.toolInitialization(false); 46 | 47 | for (final Path sstable : sstables) { 48 | final Descriptor descriptor = Descriptor.fromFilename(sstable.toAbsolutePath().toFile().getAbsolutePath()); 49 | 50 | logger.info(format("Loading file %s from initial keyspace: %s", sstable, descriptor.ksname)); 51 | 52 | final Path newSSTableDestinationDir = outputFolder.resolve(descriptor.ksname).resolve(descriptor.cfname); 53 | 54 | if (!newSSTableDestinationDir.toFile().exists()) { 55 | if (!newSSTableDestinationDir.toFile().mkdirs()) { 56 | throw new TTLRemovalException(format("Unable to create directories leading to %s.", newSSTableDestinationDir.toFile().getAbsolutePath())); 57 | } 58 | } 59 | 60 | final Descriptor resultDesc = new Descriptor(new File(newSSTableDestinationDir), 61 | descriptor.ksname, 62 | descriptor.cfname, 63 | descriptor.id, 64 | SSTableFormat.Type.BIG); 65 | 66 | final TableMetadata tableMetadata = CreateTableStatement.parse(cql, descriptor.ksname).partitioner(new Murmur3Partitioner()).build(); 67 | 68 | stream(descriptor, resultDesc, tableMetadata); 69 | } 70 | } 71 | 72 | public void stream(final Descriptor descriptor, final Descriptor toSSTable, final TableMetadata tableMetadata) throws TTLRemovalException { 73 | 74 | final SSTableReader noTTLReader; 75 | 76 | try { 77 | noTTLReader = SSTableReader.open(descriptor, SSTable.componentsFor(descriptor), TableMetadataRef.forOfflineTools(tableMetadata), true, true); 78 | } catch (final Exception ex) { 79 | throw new TTLRemovalException(format("Unable to open descriptor %s", descriptor.baseFilename()), ex); 80 | } 81 | 82 | final long keyCount = countKeys(descriptor, tableMetadata); 83 | 84 | final LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE); 85 | 86 | final SerializationHeader header = SerializationHeader.make(tableMetadata, Collections.singletonList(noTTLReader)); 87 | 88 | final SSTableRewriter writer = SSTableRewriter.constructKeepingOriginals(txn, true, Long.MAX_VALUE); 89 | 90 | writer.switchWriter(SSTableWriter.create(TableMetadataRef.forOfflineTools(tableMetadata), toSSTable, keyCount, -1, null, false, 0, header, null, txn)); 91 | 92 | try (final ISSTableScanner sourceSSTableScanner = noTTLReader.getScanner()) { 93 | while (sourceSSTableScanner.hasNext()) { 94 | final UnfilteredRowIterator partition = sourceSSTableScanner.next(); 95 | 96 | if (!partition.hasNext()) { 97 | //keep partitions with no rows 98 | writer.append(partition); 99 | continue; 100 | } 101 | 102 | final PartitionUpdate.Builder builder = new PartitionUpdate.Builder(tableMetadata, 103 | partition.partitionKey(), 104 | partition.columns(), 105 | 2, 106 | false); 107 | 108 | while (partition.hasNext()) { 109 | 110 | final Unfiltered unfiltered = partition.next(); 111 | 112 | switch (unfiltered.kind()) { 113 | case ROW: 114 | Row newRow = serializeRow(unfiltered); 115 | builder.add(newRow); 116 | break; 117 | case RANGE_TOMBSTONE_MARKER: 118 | //Range tombstones are denoted as separate (Unfiltered) entries for start and end, 119 | //so we record them separately and add the tombstone once both ends of the range are defined 120 | RangeTombstoneBoundMarker marker = (RangeTombstoneBoundMarker) unfiltered; 121 | 122 | final ClusteringBound start = marker.isOpen(false) ? marker.openBound(false) : null; 123 | final ClusteringBound end = marker.isClose(false) ? marker.closeBound(false) : null; 124 | 125 | if (start != null && end != null) { 126 | builder.add(new RangeTombstone(Slice.make(start, end), marker.deletionTime())); 127 | } 128 | 129 | break; 130 | } 131 | } 132 | 133 | writer.append(builder.build().unfilteredIterator()); 134 | } 135 | writer.finish(); 136 | } catch (final Exception ex) { 137 | throw new TTLRemovalException(format("Exception occurred while scanning SSTable %s", descriptor.baseFilename()), ex); 138 | } 139 | } 140 | 141 | private long countKeys(final Descriptor descriptor, final TableMetadata metaData) { 142 | 143 | final KeyIterator iter = new KeyIterator(descriptor, metaData); 144 | 145 | long keyCount = 0; 146 | 147 | try { 148 | while (iter.hasNext()) { 149 | iter.next(); 150 | keyCount++; 151 | } 152 | } finally { 153 | iter.close(); 154 | } 155 | 156 | return keyCount; 157 | } 158 | 159 | private Row serializeRow(final Unfiltered atoms) { 160 | 161 | final Row row = (Row) atoms; 162 | 163 | Builder builder = BTreeRow.sortedBuilder(); 164 | 165 | builder.newRow(row.clustering()); 166 | builder.addPrimaryKeyLivenessInfo(LivenessInfo.create(row.primaryKeyLivenessInfo().timestamp(), 167 | LivenessInfo.NO_TTL, 168 | FBUtilities.nowInSeconds())); 169 | 170 | row.columnData().forEach(cd -> { 171 | ColumnMetadata columnMetadata = cd.column(); 172 | if (columnMetadata.isComplex()) { 173 | for (Cell cell : row.getComplexColumnData(columnMetadata)) { 174 | builder.addCell(BufferCell.live(cell.column(), cell.timestamp(), cell.buffer(), cell.path())); 175 | } 176 | } else { 177 | Cell cell = row.getCell(columnMetadata); 178 | builder.addCell(BufferCell.live(cell.column(), cell.timestamp(), cell.buffer())); 179 | } 180 | }); 181 | builder.addRowDeletion(row.deletion()); 182 | 183 | return builder.build(); 184 | } 185 | } 186 | -------------------------------------------------------------------------------- /cassandra-3/src/main/java/com/instaclustr/cassandra/ttl/Cassandra3TTLRemover.java: -------------------------------------------------------------------------------- 1 | package com.instaclustr.cassandra.ttl; 2 | 3 | import static java.lang.String.format; 4 | 5 | import java.nio.file.Path; 6 | import java.util.*; 7 | import java.util.concurrent.TimeUnit; 8 | 9 | import com.instaclustr.cassandra.ttl.cli.TTLRemovalException; 10 | import org.apache.cassandra.config.CFMetaData; 11 | import org.apache.cassandra.config.ColumnDefinition; 12 | import org.apache.cassandra.cql3.QueryProcessor; 13 | import org.apache.cassandra.cql3.statements.CFStatement; 14 | import org.apache.cassandra.cql3.statements.CreateTableStatement; 15 | import org.apache.cassandra.db.*; 16 | import org.apache.cassandra.db.compaction.OperationType; 17 | import org.apache.cassandra.db.lifecycle.LifecycleTransaction; 18 | import org.apache.cassandra.db.marshal.CollectionType; 19 | import org.apache.cassandra.db.marshal.MapType; 20 | import org.apache.cassandra.db.partitions.PartitionUpdate; 21 | import org.apache.cassandra.db.rows.*; 22 | import org.apache.cassandra.db.rows.Row.Builder; 23 | import org.apache.cassandra.dht.Murmur3Partitioner; 24 | import org.apache.cassandra.io.sstable.Descriptor; 25 | import org.apache.cassandra.io.sstable.ISSTableScanner; 26 | import org.apache.cassandra.io.sstable.KeyIterator; 27 | import org.apache.cassandra.io.sstable.SSTable; 28 | import org.apache.cassandra.io.sstable.SSTableRewriter; 29 | import org.apache.cassandra.io.sstable.format.SSTableFormat; 30 | import org.apache.cassandra.io.sstable.format.SSTableReader; 31 | import org.apache.cassandra.io.sstable.format.SSTableWriter; 32 | import org.apache.cassandra.schema.Types; 33 | import org.apache.cassandra.utils.FBUtilities; 34 | import org.slf4j.Logger; 35 | import org.slf4j.LoggerFactory; 36 | 37 | public class Cassandra3TTLRemover implements SSTableTTLRemover { 38 | 39 | private static final Logger logger = LoggerFactory.getLogger(Cassandra3TTLRemover.class); 40 | 41 | @Override 42 | public void executeRemoval(final Path outputFolder, final Collection sstables, final String cql) throws Exception { 43 | for (final Path sstable : sstables) { 44 | 45 | final Descriptor descriptor = Descriptor.fromFilename(sstable.toAbsolutePath().toFile().getAbsolutePath()); 46 | 47 | logger.info(format("Loading file %s from initial keyspace: %s", sstable, descriptor.ksname)); 48 | 49 | final Path newSSTableDestinationDir = outputFolder.resolve(descriptor.ksname).resolve(descriptor.cfname); 50 | 51 | if (!newSSTableDestinationDir.toFile().exists()) { 52 | if (!newSSTableDestinationDir.toFile().mkdirs()) { 53 | throw new TTLRemovalException(format("Unable to create directories leading to %s.", newSSTableDestinationDir.toFile().getAbsolutePath())); 54 | } 55 | } 56 | 57 | final Descriptor resultDesc = new Descriptor(newSSTableDestinationDir.toFile(), 58 | descriptor.ksname, 59 | descriptor.cfname, 60 | descriptor.generation, 61 | SSTableFormat.Type.BIG); 62 | 63 | CFStatement parsed = (CFStatement) QueryProcessor.parseStatement(cql); 64 | parsed.prepareKeyspace(descriptor.ksname); 65 | CreateTableStatement statement = (CreateTableStatement) ((CreateTableStatement.RawStatement) parsed).prepare(Types.none()).statement; 66 | 67 | CFMetaData cfMetadata = statement.metadataBuilder() 68 | .withId(CFMetaData.generateLegacyCfId(descriptor.ksname, statement.columnFamily())) 69 | .withPartitioner(new Murmur3Partitioner()) 70 | .build() 71 | .params(statement.params()) 72 | .readRepairChance(0.0) 73 | .dcLocalReadRepairChance(0.0) 74 | .gcGraceSeconds(0) 75 | .memtableFlushPeriod((int) TimeUnit.HOURS.toMillis(1)); 76 | 77 | stream(descriptor, resultDesc, cfMetadata); 78 | } 79 | } 80 | 81 | public void stream(final Descriptor descriptor, final Descriptor toSSTable, final CFMetaData cfMetadata) throws TTLRemovalException { 82 | 83 | final SSTableReader noTTLreader; 84 | 85 | try { 86 | noTTLreader = SSTableReader.open(descriptor, SSTable.componentsFor(descriptor), cfMetadata, true, true); 87 | } catch (final Exception ex) { 88 | throw new TTLRemovalException(format("Unable to open descriptor %s", descriptor.baseFilename()), ex); 89 | } 90 | 91 | final long keyCount = countKeys(descriptor, cfMetadata); 92 | 93 | final LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE); 94 | 95 | final SerializationHeader header = SerializationHeader.make(cfMetadata, Arrays.asList(noTTLreader)); 96 | 97 | final SSTableRewriter writer = SSTableRewriter.constructKeepingOriginals(txn, true, Long.MAX_VALUE); 98 | 99 | writer.switchWriter(SSTableWriter.create(cfMetadata, toSSTable, keyCount, -1, 0, header, null, txn)); 100 | 101 | //writer.switchWriter(SSTableWriter.create(toSSTable.toString(), keyCount, -1, header, null, txn)); 102 | 103 | try (final ISSTableScanner noTTLscanner = noTTLreader.getScanner()) { 104 | while (noTTLscanner.hasNext()) { 105 | final UnfilteredRowIterator partition = noTTLscanner.next(); 106 | 107 | if (!partition.hasNext()) { 108 | //keep partitions with no rows 109 | writer.append(partition); 110 | continue; 111 | } 112 | 113 | final PartitionUpdate update = new PartitionUpdate(cfMetadata, partition.partitionKey(), partition.columns(), 2); 114 | 115 | while (partition.hasNext()) { 116 | 117 | final Unfiltered unfiltered = partition.next(); 118 | 119 | switch (unfiltered.kind()) { 120 | case ROW: 121 | Row newRow = serializeRow(unfiltered); 122 | update.add(newRow); 123 | break; 124 | case RANGE_TOMBSTONE_MARKER: 125 | //Range tombstones are denoted as separate (Unfiltered) entries for start and end, 126 | //so we record them separately and add the tombstone once both ends of the range are defined 127 | RangeTombstoneBoundMarker marker = (RangeTombstoneBoundMarker) unfiltered; 128 | 129 | final ClusteringBound start = marker.isOpen(false) ? marker.openBound(false) : null; 130 | final ClusteringBound end = marker.isClose(false) ? marker.closeBound(false) : null; 131 | 132 | if (start != null && end != null) { 133 | update.add(new RangeTombstone(Slice.make(start, end), marker.deletionTime())); 134 | } 135 | 136 | break; 137 | } 138 | } 139 | 140 | update.allowNewUpdates(); 141 | writer.append(update.unfilteredIterator()); 142 | } 143 | writer.finish(); 144 | } catch (final Exception ex) { 145 | throw new TTLRemovalException(format("Exception occurred while scanning SSTable %s", descriptor.baseFilename()), ex); 146 | } 147 | } 148 | 149 | private long countKeys(final Descriptor descriptor, final CFMetaData metaData) { 150 | 151 | final KeyIterator iter = new KeyIterator(descriptor, metaData); 152 | 153 | long keycount = 0; 154 | 155 | try { 156 | while (iter.hasNext()) { 157 | iter.next(); 158 | keycount++; 159 | } 160 | } finally { 161 | iter.close(); 162 | } 163 | 164 | return keycount; 165 | } 166 | 167 | private Row serializeRow(final Unfiltered atoms) { 168 | 169 | final Row row = (Row) atoms; 170 | 171 | Builder builder = BTreeRow.sortedBuilder(); 172 | builder.newRow(row.clustering()); 173 | 174 | builder.addPrimaryKeyLivenessInfo(LivenessInfo.create(row.primaryKeyLivenessInfo().timestamp(), 175 | LivenessInfo.NO_TTL, 176 | FBUtilities.nowInSeconds())); 177 | 178 | row.columnData().forEach(cd -> { 179 | ColumnDefinition cdef = cd.column(); 180 | if (cdef.isComplex()) { 181 | Iterator cellIterator = row.getComplexColumnData(cdef).iterator(); 182 | 183 | while (cellIterator.hasNext()) { 184 | Cell cell = cellIterator.next(); 185 | builder.addCell(BufferCell.live(cell.column(), cell.timestamp(), cell.value(), cell.path())); 186 | } 187 | } else { 188 | Cell cell = row.getCell(cdef); 189 | builder.addCell(BufferCell.live(cell.column(), cell.timestamp(), cell.value())); 190 | } 191 | }); 192 | 193 | builder.addRowDeletion(row.deletion()); 194 | 195 | return builder.build(); 196 | } 197 | } 198 | -------------------------------------------------------------------------------- /cassandra-2/src/main/java/com/instaclustr/cassandra/ttl/NoTTLSSTableNamesIterator.java: -------------------------------------------------------------------------------- 1 | package com.instaclustr.cassandra.ttl; 2 | 3 | import java.io.IOException; 4 | import java.util.*; 5 | 6 | import com.google.common.collect.AbstractIterator; 7 | 8 | import org.apache.cassandra.config.CFMetaData; 9 | import org.apache.cassandra.db.*; 10 | import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; 11 | import org.apache.cassandra.db.composites.CellName; 12 | import org.apache.cassandra.db.composites.CellNameType; 13 | import org.apache.cassandra.io.sstable.CorruptSSTableException; 14 | import org.apache.cassandra.io.sstable.IndexHelper; 15 | import org.apache.cassandra.io.util.FileDataInput; 16 | import org.apache.cassandra.io.util.FileMark; 17 | import org.apache.cassandra.io.util.FileUtils; 18 | import org.apache.cassandra.utils.ByteBufferUtil; 19 | 20 | class NoTTLSSTableNamesIterator extends AbstractIterator implements OnDiskAtomIterator 21 | { 22 | private ColumnFamily cf; 23 | private final NoTTLReader sstable; 24 | private FileDataInput fileToClose; 25 | private Iterator iter; 26 | public final SortedSet columns; 27 | public final DecoratedKey key; 28 | 29 | public NoTTLSSTableNamesIterator(NoTTLReader sstable, DecoratedKey key, SortedSet columns) 30 | { 31 | assert columns != null; 32 | this.sstable = sstable; 33 | this.columns = columns; 34 | this.key = key; 35 | 36 | RowIndexEntry indexEntry = sstable.getPosition(key, NoTTLReader.Operator.EQ); 37 | if (indexEntry == null) 38 | return; 39 | 40 | try 41 | { 42 | read(sstable, null, indexEntry); 43 | } 44 | catch (IOException e) 45 | { 46 | sstable.markSuspect(); 47 | throw new CorruptSSTableException(e, sstable.getFilename()); 48 | } 49 | finally 50 | { 51 | if (fileToClose != null) 52 | FileUtils.closeQuietly(fileToClose); 53 | } 54 | } 55 | 56 | public NoTTLSSTableNamesIterator(NoTTLReader sstable, FileDataInput file, DecoratedKey key, SortedSet columns, RowIndexEntry indexEntry) 57 | { 58 | assert columns != null; 59 | this.sstable = sstable; 60 | this.columns = columns; 61 | this.key = key; 62 | 63 | try 64 | { 65 | read(sstable, file, indexEntry); 66 | } 67 | catch (IOException e) 68 | { 69 | sstable.markSuspect(); 70 | throw new CorruptSSTableException(e, sstable.getFilename()); 71 | } 72 | } 73 | 74 | private FileDataInput createFileDataInput(long position) 75 | { 76 | fileToClose = sstable.getFileDataInput(position); 77 | return fileToClose; 78 | } 79 | 80 | @SuppressWarnings("resource") 81 | private void read(NoTTLReader sstable, FileDataInput file, RowIndexEntry indexEntry) 82 | throws IOException 83 | { 84 | List indexList; 85 | 86 | // If the entry is not indexed or the index is not promoted, read from the row start 87 | if (!indexEntry.isIndexed()) 88 | { 89 | if (file == null) 90 | file = createFileDataInput(indexEntry.position); 91 | else 92 | file.seek(indexEntry.position); 93 | 94 | DecoratedKey keyInDisk = sstable.partitioner.decorateKey(ByteBufferUtil.readWithShortLength(file)); 95 | assert keyInDisk.equals(key) : String.format("%s != %s in %s", keyInDisk, key, file.getPath()); 96 | } 97 | 98 | indexList = indexEntry.columnsIndex(); 99 | 100 | if (!indexEntry.isIndexed()) 101 | { 102 | ColumnFamilySerializer serializer = ColumnFamily.serializer; 103 | try 104 | { 105 | cf = ArrayBackedSortedColumns.factory.create(sstable.metadata); 106 | cf.delete(DeletionTime.serializer.deserialize(file)); 107 | } 108 | catch (Exception e) 109 | { 110 | throw new IOException(serializer + " failed to deserialize " + sstable.getColumnFamilyName() + " with " + sstable.metadata + " from " + file, e); 111 | } 112 | } 113 | else 114 | { 115 | cf = ArrayBackedSortedColumns.factory.create(sstable.metadata); 116 | cf.delete(indexEntry.deletionTime()); 117 | } 118 | 119 | List result = new ArrayList(); 120 | if (indexList.isEmpty()) 121 | { 122 | readSimpleColumns(file, columns, result); 123 | } 124 | else 125 | { 126 | readIndexedColumns(sstable.metadata, file, columns, indexList, indexEntry.position, result); 127 | } 128 | 129 | // create an iterator view of the columns we read 130 | iter = result.iterator(); 131 | } 132 | 133 | private void readSimpleColumns(FileDataInput file, SortedSet columnNames, List result) 134 | { 135 | Iterator atomIterator = cf.metadata().getOnDiskIterator(file, sstable.descriptor.version); 136 | int n = 0; 137 | while (atomIterator.hasNext()) 138 | { 139 | OnDiskAtom column = atomIterator.next(); 140 | if (column instanceof Cell) 141 | { 142 | if (columnNames.contains(column.name())) 143 | { 144 | result.add(column); 145 | if (++n >= columns.size()) 146 | break; 147 | } 148 | } 149 | else 150 | { 151 | result.add(column); 152 | } 153 | } 154 | } 155 | 156 | @SuppressWarnings("resource") 157 | private void readIndexedColumns(CFMetaData metadata, 158 | FileDataInput file, 159 | SortedSet columnNames, 160 | List indexList, 161 | long basePosition, 162 | List result) 163 | throws IOException 164 | { 165 | /* get the various column ranges we have to read */ 166 | CellNameType comparator = metadata.comparator; 167 | List ranges = new ArrayList(); 168 | int lastIndexIdx = -1; 169 | for (CellName name : columnNames) 170 | { 171 | int index = IndexHelper.indexFor(name, indexList, comparator, false, lastIndexIdx); 172 | if (index < 0 || index == indexList.size()) 173 | continue; 174 | IndexHelper.IndexInfo indexInfo = indexList.get(index); 175 | // Check the index block does contain the column names and that we haven't inserted this block yet. 176 | if (comparator.compare(name, indexInfo.firstName) < 0 || index == lastIndexIdx) 177 | continue; 178 | 179 | ranges.add(indexInfo); 180 | lastIndexIdx = index; 181 | } 182 | 183 | if (ranges.isEmpty()) 184 | return; 185 | 186 | Iterator toFetch = columnNames.iterator(); 187 | CellName nextToFetch = toFetch.next(); 188 | for (IndexHelper.IndexInfo indexInfo : ranges) 189 | { 190 | long positionToSeek = basePosition + indexInfo.offset; 191 | 192 | // With new promoted indexes, our first seek in the data file will happen at that point. 193 | if (file == null) 194 | file = createFileDataInput(positionToSeek); 195 | 196 | AtomDeserializer deserializer = cf.metadata().getOnDiskDeserializer(file, sstable.descriptor.version); 197 | file.seek(positionToSeek); 198 | FileMark mark = file.mark(); 199 | while (file.bytesPastMark(mark) < indexInfo.width && nextToFetch != null) 200 | { 201 | int cmp = deserializer.compareNextTo(nextToFetch); 202 | if (cmp < 0) 203 | { 204 | // If it's a rangeTombstone, then we need to read it and include 205 | // it if it includes our target. Otherwise, we can skip it. 206 | if (deserializer.nextIsRangeTombstone()) 207 | { 208 | RangeTombstone rt = (RangeTombstone)deserializer.readNext(); 209 | if (comparator.compare(rt.max, nextToFetch) >= 0) 210 | result.add(rt); 211 | } 212 | else 213 | { 214 | deserializer.skipNext(); 215 | } 216 | } 217 | else if (cmp == 0) 218 | { 219 | nextToFetch = toFetch.hasNext() ? toFetch.next() : null; 220 | result.add(deserializer.readNext()); 221 | } 222 | else 223 | nextToFetch = toFetch.hasNext() ? toFetch.next() : null; 224 | } 225 | } 226 | } 227 | 228 | public DecoratedKey getKey() 229 | { 230 | return key; 231 | } 232 | 233 | public ColumnFamily getColumnFamily() 234 | { 235 | return cf; 236 | } 237 | 238 | protected OnDiskAtom computeNext() 239 | { 240 | if (iter == null || !iter.hasNext()) 241 | return endOfData(); 242 | return iter.next(); 243 | } 244 | 245 | public void close() throws IOException { } 246 | } 247 | -------------------------------------------------------------------------------- /impl/src/main/java/com/instaclustr/cassandra/ttl/cli/TTLRemoverCLI.java: -------------------------------------------------------------------------------- 1 | package com.instaclustr.cassandra.ttl.cli; 2 | 3 | import static java.lang.String.format; 4 | import static java.util.stream.Collectors.toList; 5 | 6 | import java.io.PrintWriter; 7 | import java.nio.file.Files; 8 | import java.nio.file.Path; 9 | import java.util.Collection; 10 | import java.util.Collections; 11 | import java.util.List; 12 | import java.util.ServiceLoader; 13 | import java.util.stream.Stream; 14 | import java.util.stream.StreamSupport; 15 | 16 | import com.instaclustr.cassandra.ttl.SSTableTTLRemover; 17 | import picocli.CommandLine; 18 | import picocli.CommandLine.Command; 19 | import picocli.CommandLine.ITypeConverter; 20 | import picocli.CommandLine.Model.CommandSpec; 21 | import picocli.CommandLine.Option; 22 | import picocli.CommandLine.ParameterException; 23 | import picocli.CommandLine.Spec; 24 | 25 | @Command(name = "ttl-remove", 26 | mixinStandardHelpOptions = true, 27 | description = "command for removing TTL from SSTables", 28 | sortOptions = false, 29 | versionProvider = TTLRemoverCLI.class) 30 | public class TTLRemoverCLI extends JarManifestVersionProvider implements Runnable { 31 | 32 | @Spec 33 | protected CommandSpec spec; 34 | 35 | @Option(names = {"--output-path", "-p"}, 36 | paramLabel = "[DIRECTORY]", 37 | required = true, 38 | description = "Destination where SSTable will be generated.") 39 | protected Path destination; 40 | 41 | @Option(names = {"--cassandra-yaml", "-f"}, 42 | paramLabel = "[FILE]", 43 | description = "Path to cassandra.yaml file for loading generated SSTables to Cassandra, relevant only in case of Cassandra 2.") 44 | public Path cassandraYaml; 45 | 46 | @Option(names = {"--cassandra-storage-dir", "-d"}, 47 | paramLabel = "[DIRECTORY]", 48 | description = "Path to cassandra data dir, relevant only in case of Cassandra 2.") 49 | public Path cassandraStorageDir; 50 | 51 | @Option(names = {"--sstables", "-s"}, 52 | paramLabel = "[DIRECTORY]", 53 | description = "Path to a directory for which all SSTables will have TTL removed.") 54 | public Path sstables; 55 | 56 | @Option(names = {"--sstable", "-t"}, 57 | paramLabel = "[FILE]", 58 | description = "Path to .db file of a SSTable which will have TTL removed.") 59 | public Path sstable; 60 | 61 | @Option(names = {"--cassandra-version", "-c"}, 62 | paramLabel = "[INTEGER]", 63 | converter = CassandraVersionConverter.class, 64 | description = "Version of Cassandra to remove TTL for, might be 2, 3 or 4, defaults to 3") 65 | public CassandraVersion cassandraVersion; 66 | 67 | @Option(names = {"--cql", "-q"}, 68 | paramLabel = "[CQL]", 69 | description = "CQL statement which creates table we want to remove TTL from. This has to be set in case --cassandra-version is 3 or 4") 70 | public String cql; 71 | 72 | public static void main(String[] args) { 73 | main(args, true); 74 | } 75 | 76 | public static void main(String[] args, boolean exit) { 77 | int exitCode = execute(new CommandLine(new TTLRemoverCLI()), args); 78 | 79 | if (exit) { 80 | System.exit(exitCode); 81 | } 82 | } 83 | 84 | @Override 85 | public String getImplementationTitle() { 86 | return "ttl-remove"; 87 | } 88 | 89 | @Override 90 | public void run() { 91 | JarManifestVersionProvider.logCommandVersionInformation(spec); 92 | 93 | validate(); 94 | 95 | if (!Boolean.parseBoolean(System.getProperty("ttl.remover.tests", "false"))) { 96 | TTLRemoverCLI.setProperties(cassandraYaml, cassandraStorageDir, cassandraVersion); 97 | } 98 | 99 | try { 100 | final SSTableTTLRemover ttlRemover = getTTLRemover(); 101 | ttlRemover.executeRemoval(destination, getSSTables(), cql); 102 | } catch (final Exception ex) { 103 | throw new RuntimeException("Unable to remove TTLs from SSTables", ex); 104 | } 105 | } 106 | 107 | public static int execute(CommandLine commandLine, String... args) { 108 | return commandLine 109 | .setErr(new PrintWriter(System.err)) 110 | .setOut(new PrintWriter(System.err)) 111 | .setUnmatchedArgumentsAllowed(false) 112 | .setColorScheme(new CommandLine.Help.ColorScheme.Builder().ansi(CommandLine.Help.Ansi.ON).build()) 113 | .setExecutionExceptionHandler((ex, cmdLine, parseResult) -> { 114 | 115 | ex.printStackTrace(); 116 | 117 | return 1; 118 | }) 119 | .execute(args); 120 | } 121 | 122 | 123 | // relevant for Cassandra 2 only 124 | public static void setProperties(final Path cassandraYaml, final Path cassandraStorageDir, final CassandraVersion version) { 125 | if (version == CassandraVersion.V2) { 126 | System.setProperty("cassandra.config", "file://" + cassandraYaml.toAbsolutePath().toString()); 127 | System.setProperty("cassandra.storagedir", cassandraStorageDir.toAbsolutePath().toString()); 128 | } 129 | } 130 | 131 | private SSTableTTLRemover getTTLRemover() throws TTLRemovalException { 132 | final ServiceLoader serviceLoader = ServiceLoader.load(SSTableTTLRemover.class); 133 | 134 | final List removers = StreamSupport.stream(serviceLoader.spliterator(), false).collect(toList()); 135 | 136 | if (removers.size() == 0) { 137 | throw new TTLRemovalException("Unable to locate an instance of SSTableTTLRemover on the class path."); 138 | } else if (removers.size() != 1) { 139 | throw new TTLRemovalException(format("There is %s implementations of %s on the class path, there needs to be just one!", 140 | removers.size(), 141 | SSTableTTLRemover.class.getName())); 142 | } 143 | 144 | return removers.get(0); 145 | } 146 | 147 | private Collection getSSTables() throws TTLRemovalException { 148 | if (sstables != null) { 149 | try (final Stream stream = Files.walk(sstables)) { 150 | return stream.filter(f -> f.toString().endsWith("Data.db")).collect(toList()); 151 | } catch (final Exception ex) { 152 | throw new RuntimeException(format("Unable to walk keyspace directory %s", sstables), ex); 153 | } 154 | } else if (sstable != null) { 155 | if (!sstable.toFile().exists() || sstable.toFile().canRead()) { 156 | throw new RuntimeException(format("SSTable %s does not exist or it can not be read.", sstable)); 157 | } 158 | 159 | return Collections.singleton(sstable); 160 | } 161 | 162 | throw new TTLRemovalException("--sstables nor --sstable parameter was set, you have to set one of them!"); 163 | } 164 | 165 | private void validate() { 166 | if (cassandraVersion != CassandraVersion.V2 && cql == null) { 167 | throw new ParameterException(spec.commandLine(), 168 | format("You want to remove TTL from SSTables for version %s but you have not specified --cql", 169 | cassandraVersion)); 170 | } 171 | 172 | if (cassandraVersion == CassandraVersion.V2) { 173 | if (cassandraYaml == null) { 174 | throw new ParameterException(spec.commandLine(), "You set Cassandra version to '2' but you have not set --cassandra-yaml"); 175 | } 176 | if (cassandraStorageDir == null) { 177 | throw new ParameterException(spec.commandLine(), "You set Cassandra version to '2' but you have not set --cassandra-storage-dir"); 178 | } 179 | } 180 | 181 | if (cassandraVersion != CassandraVersion.V2) { 182 | if (cassandraYaml != null) { 183 | throw new ParameterException(spec.commandLine(), format("You set Cassandra version to '%s' but you have set --cassandra-yaml", cassandraVersion)); 184 | } 185 | if (cassandraStorageDir != null) { 186 | throw new ParameterException(spec.commandLine(), format("You set Cassandra version to '%s' but you have set --cassandra-storage-dir", cassandraVersion)); 187 | } 188 | } 189 | 190 | if (sstables == null && sstable == null) { 191 | throw new ParameterException(spec.commandLine(), "You have not specified --sstables nor --sstable."); 192 | } 193 | 194 | if (sstables != null && sstable != null) { 195 | throw new ParameterException(spec.commandLine(), "You have specified both --sstables and --sstable."); 196 | } 197 | } 198 | 199 | private static final class CassandraVersionConverter implements ITypeConverter { 200 | 201 | @Override 202 | public CassandraVersion convert(final String value) { 203 | return CassandraVersion.parse(value); 204 | } 205 | } 206 | 207 | public enum CassandraVersion { 208 | V2("2"), 209 | V3("3"), 210 | V4("4"); 211 | 212 | final String version; 213 | 214 | CassandraVersion(final String version) { 215 | this.version = version; 216 | } 217 | 218 | public static CassandraVersion parse(final String version) { 219 | if (version == null) { 220 | return V3; 221 | } 222 | 223 | if (version.equals("2")) { 224 | return V2; 225 | } 226 | 227 | if (version.equals("3")) { 228 | return V3; 229 | } 230 | 231 | if (version.equals("4")) { 232 | return V4; 233 | } 234 | 235 | return V3; 236 | } 237 | } 238 | } -------------------------------------------------------------------------------- /cassandra-2/src/main/java/com/instaclustr/cassandra/ttl/NoTTLScanner.java: -------------------------------------------------------------------------------- 1 | package com.instaclustr.cassandra.ttl; 2 | 3 | 4 | import static org.apache.cassandra.dht.AbstractBounds.isEmpty; 5 | import static org.apache.cassandra.dht.AbstractBounds.maxLeft; 6 | import static org.apache.cassandra.dht.AbstractBounds.minRight; 7 | 8 | import java.io.IOException; 9 | import java.util.ArrayList; 10 | import java.util.Iterator; 11 | import java.util.List; 12 | import java.util.concurrent.atomic.AtomicBoolean; 13 | 14 | import com.google.common.collect.AbstractIterator; 15 | import com.google.common.util.concurrent.RateLimiter; 16 | import org.apache.cassandra.db.DataRange; 17 | import org.apache.cassandra.db.DecoratedKey; 18 | import org.apache.cassandra.db.RowIndexEntry; 19 | import org.apache.cassandra.db.RowPosition; 20 | import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; 21 | import org.apache.cassandra.dht.AbstractBounds; 22 | import org.apache.cassandra.dht.Range; 23 | import org.apache.cassandra.io.sstable.CorruptSSTableException; 24 | import org.apache.cassandra.io.sstable.ISSTableScanner; 25 | import org.apache.cassandra.io.util.FileUtils; 26 | import org.apache.cassandra.io.util.RandomAccessReader; 27 | import org.apache.cassandra.utils.ByteBufferUtil; 28 | 29 | /** 30 | * Cannot extend BigTableScanner because its constructor is private. 31 | */ 32 | public class NoTTLScanner implements ISSTableScanner { 33 | 34 | private AtomicBoolean isClosed = new AtomicBoolean(false); 35 | protected final RandomAccessReader dfile; 36 | protected final RandomAccessReader ifile; 37 | public final NoTTLReader sstable; 38 | 39 | private final Iterator> rangeIterator; 40 | private AbstractBounds currentRange; 41 | 42 | private final DataRange dataRange; 43 | private final RowIndexEntry.IndexSerializer rowIndexEntrySerializer; 44 | 45 | protected Iterator iterator; 46 | 47 | public static ISSTableScanner getScanner(NoTTLReader sstable, DataRange dataRange, RateLimiter limiter) { 48 | return new NoTTLScanner(sstable, dataRange, limiter); 49 | } 50 | 51 | 52 | private NoTTLScanner(NoTTLReader sstable, DataRange dataRange, RateLimiter limiter) { 53 | assert sstable != null; 54 | 55 | this.dfile = limiter == null ? sstable.openDataReader() : sstable.openDataReader(limiter); 56 | this.ifile = sstable.openIndexReader(); 57 | this.sstable = sstable; 58 | this.dataRange = dataRange; 59 | this.rowIndexEntrySerializer = sstable.descriptor.version.getSSTableFormat().getIndexSerializer(sstable.metadata); 60 | 61 | List> boundsList = new ArrayList<>(2); 62 | addRange(dataRange.keyRange(), boundsList); 63 | this.rangeIterator = boundsList.iterator(); 64 | } 65 | 66 | private void addRange(AbstractBounds requested, List> boundsList) { 67 | if (requested instanceof Range && ((Range) requested).isWrapAround()) { 68 | if (requested.right.compareTo(sstable.first) >= 0) { 69 | // since we wrap, we must contain the whole sstable prior to stopKey() 70 | AbstractBounds.Boundary left = new AbstractBounds.Boundary(sstable.first, true); 71 | AbstractBounds.Boundary right; 72 | right = requested.rightBoundary(); 73 | right = minRight(right, sstable.last, true); 74 | if (!isEmpty(left, right)) { 75 | boundsList.add(AbstractBounds.bounds(left, right)); 76 | } 77 | } 78 | if (requested.left.compareTo(sstable.last) <= 0) { 79 | // since we wrap, we must contain the whole sstable after dataRange.startKey() 80 | AbstractBounds.Boundary right = new AbstractBounds.Boundary(sstable.last, true); 81 | AbstractBounds.Boundary left; 82 | left = requested.leftBoundary(); 83 | left = maxLeft(left, sstable.first, true); 84 | if (!isEmpty(left, right)) { 85 | boundsList.add(AbstractBounds.bounds(left, right)); 86 | } 87 | } 88 | } else { 89 | assert requested.left.compareTo(requested.right) <= 0 || requested.right.isMinimum(); 90 | AbstractBounds.Boundary left, right; 91 | left = requested.leftBoundary(); 92 | right = requested.rightBoundary(); 93 | left = maxLeft(left, sstable.first, true); 94 | // apparently isWrapAround() doesn't count Bounds that extend to the limit (min) as wrapping 95 | right = requested.right.isMinimum() ? new AbstractBounds.Boundary(sstable.last, true) 96 | : minRight(right, sstable.last, true); 97 | if (!isEmpty(left, right)) { 98 | boundsList.add(AbstractBounds.bounds(left, right)); 99 | } 100 | } 101 | } 102 | 103 | public void close() throws IOException { 104 | if (isClosed.compareAndSet(false, true)) { 105 | FileUtils.close(dfile, ifile); 106 | } 107 | } 108 | 109 | public long getLengthInBytes() { 110 | return dfile.length(); 111 | } 112 | 113 | public long getCurrentPosition() { 114 | return dfile.getFilePointer(); 115 | } 116 | 117 | public String getBackingFiles() { 118 | return sstable.toString(); 119 | } 120 | 121 | public boolean hasNext() { 122 | if (iterator == null) { 123 | iterator = createIterator(); 124 | } 125 | return iterator.hasNext(); 126 | } 127 | 128 | public OnDiskAtomIterator next() { 129 | if (iterator == null) // iterator is not null 130 | { 131 | iterator = createIterator(); 132 | } 133 | return iterator.next(); 134 | } 135 | 136 | private Iterator createIterator() { 137 | return new NoTTLKeyScanningIterator(); 138 | } 139 | 140 | private void seekToCurrentRangeStart() { 141 | long indexPosition = sstable.getIndexScanPosition(currentRange.left); 142 | ifile.seek(indexPosition); 143 | try { 144 | 145 | while (!ifile.isEOF()) { 146 | indexPosition = ifile.getFilePointer(); 147 | DecoratedKey indexDecoratedKey = sstable.partitioner.decorateKey(ByteBufferUtil.readWithShortLength(ifile)); 148 | if (indexDecoratedKey.compareTo(currentRange.left) > 0 || currentRange.contains(indexDecoratedKey)) { 149 | // Found, just read the dataPosition and seek into index and data files 150 | long dataPosition = ifile.readLong(); 151 | ifile.seek(indexPosition); 152 | dfile.seek(dataPosition); 153 | break; 154 | } else { 155 | RowIndexEntry.Serializer.skip(ifile); 156 | } 157 | } 158 | } catch (IOException e) { 159 | sstable.markSuspect(); 160 | throw new CorruptSSTableException(e, sstable.getFilename()); 161 | } 162 | } 163 | 164 | protected class NoTTLKeyScanningIterator extends AbstractIterator { 165 | 166 | private DecoratedKey nextKey; 167 | private RowIndexEntry nextEntry; 168 | private DecoratedKey currentKey; 169 | private RowIndexEntry currentEntry; 170 | 171 | protected OnDiskAtomIterator computeNext() { 172 | try { 173 | if (nextEntry == null) { 174 | do { 175 | // we're starting the first range or we just passed the end of the previous range 176 | if (!rangeIterator.hasNext()) { 177 | return endOfData(); 178 | } 179 | 180 | currentRange = rangeIterator.next(); 181 | seekToCurrentRangeStart(); 182 | 183 | if (ifile.isEOF()) { 184 | return endOfData(); 185 | } 186 | 187 | currentKey = sstable.partitioner.decorateKey(ByteBufferUtil.readWithShortLength(ifile)); 188 | currentEntry = rowIndexEntrySerializer.deserialize(ifile, sstable.descriptor.version); 189 | } while (!currentRange.contains(currentKey)); 190 | } else { 191 | // we're in the middle of a range 192 | currentKey = nextKey; 193 | currentEntry = nextEntry; 194 | } 195 | 196 | if (ifile.isEOF()) { 197 | nextEntry = null; 198 | nextKey = null; 199 | } else { 200 | // we need the position of the start of the next key, regardless of whether it falls in the current range 201 | nextKey = sstable.partitioner.decorateKey(ByteBufferUtil.readWithShortLength(ifile)); 202 | nextEntry = rowIndexEntrySerializer.deserialize(ifile, sstable.descriptor.version); 203 | 204 | if (!currentRange.contains(nextKey)) { 205 | nextKey = null; 206 | nextEntry = null; 207 | } 208 | } 209 | 210 | dfile.seek(currentEntry.position + currentEntry.headerOffset()); 211 | ByteBufferUtil.readWithShortLength(dfile); // key 212 | return new NoTTLSSTableIdentityIterator(sstable, dfile, currentKey, false); 213 | 214 | 215 | } catch (CorruptSSTableException | IOException e) { 216 | sstable.markSuspect(); 217 | throw new CorruptSSTableException(e, sstable.getFilename()); 218 | } 219 | } 220 | } 221 | 222 | } 223 | -------------------------------------------------------------------------------- /buddy-agent/src/main/java/com/instaclustr/cassandra/ttl/buddy/CassandraAgent.java: -------------------------------------------------------------------------------- 1 | package com.instaclustr.cassandra.ttl.buddy; 2 | 3 | import java.lang.instrument.Instrumentation; 4 | 5 | import net.bytebuddy.agent.builder.AgentBuilder.Default; 6 | import net.bytebuddy.implementation.FixedValue; 7 | import net.bytebuddy.matcher.ElementMatchers; 8 | import org.apache.cassandra.config.Config.CorruptedTombstoneStrategy; 9 | import org.apache.cassandra.config.Config.DiskAccessMode; 10 | import org.apache.cassandra.io.util.SpinningDiskOptimizationStrategy; 11 | import org.apache.cassandra.metrics.RestorableMeter; 12 | 13 | public class CassandraAgent { 14 | 15 | public static void premain(String arg, Instrumentation inst) { 16 | 17 | final Default agentBuilder = new Default(); 18 | 19 | agentBuilder 20 | .type(ElementMatchers.named("org.apache.cassandra.config.DatabaseDescriptor")) 21 | .transform((builder, typeDescription, classLoader, javaModule) -> 22 | builder.method(ElementMatchers.named("getMinRpcTimeout")).intercept(FixedValue.value(5000))) 23 | .installOn(inst); 24 | 25 | agentBuilder 26 | .type(ElementMatchers.named("org.apache.cassandra.config.DatabaseDescriptor")) 27 | .transform((builder, typeDescription, classLoader, javaModule) -> 28 | builder.method(ElementMatchers.named("getTruncateRpcTimeout")).intercept(FixedValue.value(60000))) 29 | .installOn(inst); 30 | 31 | agentBuilder 32 | .type(ElementMatchers.named("org.apache.cassandra.config.DatabaseDescriptor")) 33 | .transform((builder, typeDescription, classLoader, javaModule) -> 34 | builder.method(ElementMatchers.named("getCounterWriteRpcTimeout")).intercept(FixedValue.value(5000))) 35 | .installOn(inst); 36 | 37 | agentBuilder 38 | .type(ElementMatchers.named("org.apache.cassandra.config.DatabaseDescriptor")) 39 | .transform((builder, typeDescription, classLoader, javaModule) -> 40 | builder.method(ElementMatchers.named("getWriteRpcTimeout")).intercept(FixedValue.value(2000))) 41 | .installOn(inst); 42 | 43 | agentBuilder 44 | .type(ElementMatchers.named("org.apache.cassandra.config.DatabaseDescriptor")) 45 | .transform((builder, typeDescription, classLoader, javaModule) -> 46 | builder.method(ElementMatchers.named("getRangeRpcTimeout")).intercept(FixedValue.value(10000))) 47 | .installOn(inst); 48 | 49 | agentBuilder 50 | .type(ElementMatchers.named("org.apache.cassandra.config.DatabaseDescriptor")) 51 | .transform((builder, typeDescription, classLoader, javaModule) -> 52 | builder.method(ElementMatchers.named("getReadRpcTimeout")).intercept(FixedValue.value(5000))) 53 | .installOn(inst); 54 | 55 | agentBuilder 56 | .type(ElementMatchers.named("org.apache.cassandra.config.DatabaseDescriptor")) 57 | .transform((builder, typeDescription, classLoader, javaModule) -> 58 | builder.method(ElementMatchers.named("getRpcTimeout")).intercept(FixedValue.value(5000))) 59 | .installOn(inst); 60 | 61 | agentBuilder 62 | .type(ElementMatchers.named("org.apache.cassandra.config.DatabaseDescriptor")) 63 | .transform((builder, typeDescription, classLoader, javaModule) -> 64 | builder.method(ElementMatchers.named("getConcurrentCounterWriters")).intercept(FixedValue.value(32))) 65 | .installOn(inst); 66 | 67 | agentBuilder 68 | .type(ElementMatchers.named("org.apache.cassandra.config.DatabaseDescriptor")) 69 | .transform((builder, typeDescription, classLoader, javaModule) -> 70 | builder.method(ElementMatchers.named("getBufferPoolUseHeapIfExhausted")).intercept(FixedValue.value(true))) 71 | .installOn(inst); 72 | 73 | agentBuilder 74 | .type(ElementMatchers.named("org.apache.cassandra.db.SystemKeyspace")) 75 | .transform((builder, typeDescription, classLoader, javaModule) -> 76 | builder.method(ElementMatchers.named("getSSTableReadMeter")).intercept(FixedValue.value(new RestorableMeter()))) 77 | .installOn(inst); 78 | 79 | agentBuilder 80 | .type(ElementMatchers.named("org.apache.cassandra.config.DatabaseDescriptor")) 81 | .transform((builder, typeDescription, classLoader, javaModule) -> 82 | builder.method(ElementMatchers.named("shouldMigrateKeycacheOnCompaction")).intercept(FixedValue.value(true))) 83 | .installOn(inst); 84 | 85 | agentBuilder 86 | .type(ElementMatchers.named("org.apache.cassandra.config.DatabaseDescriptor")) 87 | .transform((builder, typeDescription, classLoader, javaModule) -> 88 | builder.method(ElementMatchers.named("getCompactionLargePartitionWarningThreshold")).intercept(FixedValue.value(100 * 1024 * 1024))) 89 | .installOn(inst); 90 | 91 | agentBuilder 92 | .type(ElementMatchers.named("org.apache.cassandra.config.DatabaseDescriptor")) 93 | .transform((builder, typeDescription, classLoader, javaModule) -> 94 | builder.method(ElementMatchers.named("getColumnIndexSize")).intercept(FixedValue.value(64 * 1024))) 95 | .installOn(inst); 96 | 97 | agentBuilder 98 | .type(ElementMatchers.named("org.apache.cassandra.config.DatabaseDescriptor")) 99 | .transform((builder, typeDescription, classLoader, javaModule) -> 100 | builder.method(ElementMatchers.named("getColumnIndexCacheSize")).intercept(FixedValue.value(2 * 1024))) 101 | .installOn(inst); 102 | 103 | agentBuilder 104 | .type(ElementMatchers.named("org.apache.cassandra.config.DatabaseDescriptor")) 105 | .transform((builder, typeDescription, classLoader, javaModule) -> 106 | builder.method(ElementMatchers.named("getCorruptedTombstoneStrategy")).intercept(FixedValue.value(CorruptedTombstoneStrategy.warn))) 107 | .installOn(inst); 108 | 109 | agentBuilder 110 | .type(ElementMatchers.named("org.apache.cassandra.config.DatabaseDescriptor")) 111 | .transform((builder, typeDescription, classLoader, javaModule) -> 112 | builder.method(ElementMatchers.named("getMaxValueSize")).intercept(FixedValue.value(256 * 1024 * 1024))) 113 | .installOn(inst); 114 | 115 | agentBuilder 116 | .type(ElementMatchers.named("org.apache.cassandra.config.DatabaseDescriptor")) 117 | .transform((builder, typeDescription, classLoader, javaModule) -> 118 | builder.method(ElementMatchers.named("getTrickleFsyncIntervalInKb")).intercept(FixedValue.value(10240))) 119 | .installOn(inst); 120 | 121 | agentBuilder 122 | .type(ElementMatchers.named("org.apache.cassandra.config.DatabaseDescriptor")) 123 | .transform((builder, typeDescription, classLoader, javaModule) -> 124 | builder.method(ElementMatchers.named("getTrickleFsync")).intercept(FixedValue.value(false))) 125 | .installOn(inst); 126 | 127 | agentBuilder 128 | .type(ElementMatchers.named("org.apache.cassandra.config.DatabaseDescriptor")) 129 | .transform((builder, typeDescription, classLoader, javaModule) -> 130 | builder.method(ElementMatchers.named("getDiskAccessMode")).intercept(FixedValue.value(DiskAccessMode.standard))) 131 | .installOn(inst); 132 | 133 | agentBuilder 134 | .type(ElementMatchers.named("org.apache.cassandra.config.DatabaseDescriptor")) 135 | .transform((builder, typeDescription, classLoader, javaModule) -> 136 | builder.method(ElementMatchers.named("getSSTablePreemptiveOpenIntervalInMB")).intercept(FixedValue.value(50))) 137 | .installOn(inst); 138 | 139 | // on 3.11.x, there is typo - check "Preempive" in that method 140 | agentBuilder 141 | .type(ElementMatchers.named("org.apache.cassandra.config.DatabaseDescriptor")) 142 | .transform((builder, typeDescription, classLoader, javaModule) -> 143 | builder.method(ElementMatchers.named("getSSTablePreempiveOpenIntervalInMB")).intercept(FixedValue.value(50))) 144 | .installOn(inst); 145 | 146 | agentBuilder 147 | .type(ElementMatchers.named("org.apache.cassandra.config.DatabaseDescriptor")) 148 | .transform((builder, typeDescription, classLoader, javaModule) -> 149 | builder.method(ElementMatchers.named("getDiskOptimizationEstimatePercentile")).intercept(FixedValue.value(0.95))) 150 | .installOn(inst); 151 | 152 | agentBuilder 153 | .type(ElementMatchers.named("org.apache.cassandra.config.DatabaseDescriptor")) 154 | .transform((builder, typeDescription, classLoader, javaModule) -> 155 | builder.method(ElementMatchers.named("getFileCacheRoundUp")).intercept(FixedValue.value(false))) 156 | .installOn(inst); 157 | 158 | agentBuilder 159 | .type(ElementMatchers.named("org.apache.cassandra.config.DatabaseDescriptor")) 160 | .transform((builder, typeDescription, classLoader, javaModule) -> 161 | builder.method(ElementMatchers.named("getLocalSystemKeyspacesDataFileLocations")).intercept(FixedValue.value(new String[]{}))) 162 | .installOn(inst); 163 | 164 | agentBuilder 165 | .type(ElementMatchers.named("org.apache.cassandra.config.DatabaseDescriptor")) 166 | .transform((builder, typeDescription, classLoader, javaModule) -> 167 | builder.method(ElementMatchers.named("getNonLocalSystemKeyspacesDataFileLocations")).intercept(FixedValue.value(FixedValue.value(new String[]{})))) 168 | .installOn(inst); 169 | 170 | agentBuilder 171 | .type(ElementMatchers.named("org.apache.cassandra.config.DatabaseDescriptor")) 172 | .transform((builder, typeDescription, classLoader, javaModule) -> 173 | builder.method(ElementMatchers.named("getAllDataFileLocations")).intercept(FixedValue.value(new String[]{}))) 174 | .installOn(inst); 175 | 176 | agentBuilder 177 | .type(ElementMatchers.named("org.apache.cassandra.config.DatabaseDescriptor")) 178 | .transform((builder, typeDescription, classLoader, javaModule) -> 179 | builder.method(ElementMatchers.named("getFileCacheSizeInMB")).intercept(FixedValue.value(1))) 180 | .installOn(inst); 181 | 182 | agentBuilder 183 | .type(ElementMatchers.named("org.apache.cassandra.config.DatabaseDescriptor")) 184 | .transform((builder, typeDescription, classLoader, javaModule) -> 185 | builder.method(ElementMatchers.named("getDiskOptimizationStrategy")).intercept(FixedValue.value(new SpinningDiskOptimizationStrategy()))) 186 | .installOn(inst); 187 | 188 | agentBuilder 189 | .type(ElementMatchers.named("org.apache.cassandra.config.DatabaseDescriptor")) 190 | .transform((builder, typeDescription, classLoader, javaModule) -> 191 | builder.method(ElementMatchers.named("getNetworkingCacheSizeInMB")).intercept(FixedValue.value(0))) 192 | .installOn(inst); 193 | 194 | agentBuilder 195 | .type(ElementMatchers.named("org.apache.cassandra.config.DatabaseDescriptor")) 196 | .transform((builder, typeDescription, classLoader, javaModule) -> 197 | builder.method(ElementMatchers.named("getFileCacheEnabled")).intercept(FixedValue.value(false))) 198 | .installOn(inst); 199 | } 200 | } 201 | -------------------------------------------------------------------------------- /cassandra-4.1/src/test/java/com/instaclustr/cassandra/ttl/Cassandra41TTLRemoverTest.java: -------------------------------------------------------------------------------- 1 | package com.instaclustr.cassandra.ttl; 2 | 3 | import static java.lang.String.format; 4 | import static org.awaitility.Awaitility.await; 5 | import static org.junit.Assert.assertEquals; 6 | 7 | import java.io.File; 8 | import java.nio.file.Files; 9 | import java.nio.file.Path; 10 | import java.nio.file.Paths; 11 | import java.util.ArrayList; 12 | import java.util.List; 13 | import java.util.Objects; 14 | import java.util.UUID; 15 | import java.util.concurrent.TimeUnit; 16 | import java.util.function.Consumer; 17 | import java.util.stream.Stream; 18 | 19 | import com.datastax.driver.core.Cluster; 20 | import com.datastax.driver.core.Row; 21 | import com.datastax.driver.core.Session; 22 | import com.datastax.driver.core.querybuilder.QueryBuilder; 23 | import com.github.nosan.embedded.cassandra.Cassandra; 24 | import com.github.nosan.embedded.cassandra.CassandraBuilder; 25 | import com.github.nosan.embedded.cassandra.Version; 26 | import com.github.nosan.embedded.cassandra.WorkingDirectoryDestroyer; 27 | import com.instaclustr.cassandra.ttl.cli.TTLRemoverCLI; 28 | import com.instaclustr.sstable.generator.BulkLoader; 29 | import com.instaclustr.sstable.generator.CassandraBulkLoader; 30 | import com.instaclustr.sstable.generator.Generator; 31 | import com.instaclustr.sstable.generator.MappedRow; 32 | import com.instaclustr.sstable.generator.RowMapper; 33 | import com.instaclustr.sstable.generator.SSTableGenerator; 34 | import com.instaclustr.sstable.generator.cli.CLIApplication; 35 | import com.instaclustr.sstable.generator.exception.SSTableGeneratorException; 36 | import com.instaclustr.sstable.generator.specs.BulkLoaderSpec; 37 | import com.instaclustr.sstable.generator.specs.CassandraBulkLoaderSpec; 38 | import com.instaclustr.sstable.generator.specs.CassandraBulkLoaderSpec.CassandraVersion; 39 | import org.apache.cassandra.config.DatabaseDescriptor; 40 | import org.apache.cassandra.tools.Cassandra41CustomBulkLoader; 41 | import org.junit.Rule; 42 | import org.junit.Test; 43 | import org.junit.rules.TemporaryFolder; 44 | import org.junit.runner.RunWith; 45 | import org.junit.runners.JUnit4; 46 | import org.slf4j.Logger; 47 | import org.slf4j.LoggerFactory; 48 | import picocli.CommandLine.Command; 49 | 50 | @RunWith(JUnit4.class) 51 | public class Cassandra41TTLRemoverTest { 52 | 53 | private static final Logger logger = LoggerFactory.getLogger(Cassandra41TTLRemoverTest.class); 54 | 55 | private static final String CASSANDRA_VERSION = System.getProperty("version.cassandra41", "4.1.0"); 56 | private static final Path cassandraDir = new File("target/cassandra-4.1").toPath().toAbsolutePath(); 57 | 58 | private static final String KEYSPACE = "test"; 59 | private static final String TABLE = "test"; 60 | 61 | @Rule 62 | public TemporaryFolder noTTLSSTables = new TemporaryFolder(); 63 | 64 | @Rule 65 | public TemporaryFolder generatedSSTables = new TemporaryFolder(); 66 | 67 | 68 | @Test 69 | public void removeTTL() throws InterruptedException { 70 | 71 | logger.info(System.getProperty("java.library.path")); 72 | 73 | Path cassandraDir = new File("target/cassandra-4.1").toPath().toAbsolutePath(); 74 | 75 | Cassandra cassandra = getCassandra(); 76 | 77 | try { 78 | cassandra.start(); 79 | 80 | waitForCql(); 81 | 82 | executeWithSession(session -> { 83 | session.execute(String.format("CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1 };", KEYSPACE)); 84 | session.execute(String.format("CREATE TABLE IF NOT EXISTS %s.%s (id uuid, name text, surname text, PRIMARY KEY (id)) WITH default_time_to_live = 10;", KEYSPACE, TABLE)); 85 | }); 86 | 87 | // this has to be here for streaming in loader ... yeah, just here 88 | System.setProperty("cassandra.storagedir", cassandraDir.resolve("data").toAbsolutePath().toString()); 89 | System.setProperty("cassandra.config", "file://" + findCassandraYaml(cassandraDir.resolve("conf")).toAbsolutePath()); 90 | DatabaseDescriptor.toolInitialization(false); 91 | 92 | final BulkLoaderSpec bulkLoaderSpec = new BulkLoaderSpec(); 93 | 94 | bulkLoaderSpec.bufferSize = 128; 95 | bulkLoaderSpec.file = Paths.get(""); 96 | bulkLoaderSpec.keyspace = KEYSPACE; 97 | bulkLoaderSpec.table = TABLE; 98 | bulkLoaderSpec.partitioner = "murmur"; 99 | bulkLoaderSpec.sorted = false; 100 | bulkLoaderSpec.threads = 1; 101 | 102 | bulkLoaderSpec.generationImplementation = TestFixedImplementation.class.getName(); 103 | bulkLoaderSpec.outputDir = generatedSSTables.getRoot().toPath(); 104 | bulkLoaderSpec.schema = Paths.get(new File("src/test/resources/cassandra/cql/table.cql").getAbsolutePath()); 105 | 106 | final BulkLoader bulkLoader = new TestBulkLoader(); 107 | bulkLoader.bulkLoaderSpec = bulkLoaderSpec; 108 | 109 | bulkLoader.run(); 110 | 111 | // wait until data would expire 112 | Thread.sleep(15000); 113 | 114 | // here we see they expired 115 | 116 | try (final Cluster cluster = Cluster.builder().addContactPoint("127.0.0.1").withPort(9042).build(); final Session session = cluster.connect()) { 117 | assertEquals(0, session.execute(QueryBuilder.select().all().from("test", "test")).all().size()); 118 | } 119 | 120 | cassandra.stop(); 121 | 122 | logger.info("Removing TTLs ..."); 123 | 124 | // remove ttls 125 | 126 | TTLRemoverCLI.main(new String[]{ 127 | "--cassandra-version=4", 128 | "--sstables", 129 | bulkLoaderSpec.outputDir.toAbsolutePath() + "/test", 130 | "--output-path", 131 | noTTLSSTables.getRoot().toPath().toString(), 132 | "--cql", 133 | "CREATE TABLE IF NOT EXISTS test.test (id uuid, name text, surname text, PRIMARY KEY (id)) WITH default_time_to_live = 10;" 134 | }, false); 135 | 136 | // start new Cassandra instance 137 | 138 | cassandra.start(); 139 | 140 | executeWithSession(session -> { 141 | session.execute(String.format("CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1 };", KEYSPACE)); 142 | session.execute(String.format("CREATE TABLE IF NOT EXISTS %s.%s (id uuid, name text, surname text, PRIMARY KEY (id)) WITH default_time_to_live = 10;", KEYSPACE, TABLE)); 143 | }); 144 | 145 | final CassandraBulkLoader cassandraBulkLoader2 = new Cassandra41CustomBulkLoader(); 146 | 147 | final CassandraBulkLoaderSpec cassandraBulkLoaderSpec2 = new CassandraBulkLoaderSpec(); 148 | 149 | cassandraBulkLoaderSpec2.node = "127.0.0.1"; 150 | cassandraBulkLoaderSpec2.cassandraYaml = findCassandraYaml(cassandraDir.resolve("conf")); 151 | cassandraBulkLoaderSpec2.sstablesDir = Paths.get(noTTLSSTables.getRoot().getAbsolutePath(), KEYSPACE, TABLE); 152 | cassandraBulkLoaderSpec2.cassandraVersion = CassandraVersion.V4; 153 | cassandraBulkLoaderSpec2.keyspace = KEYSPACE; 154 | 155 | cassandraBulkLoader2.cassandraBulkLoaderSpec = cassandraBulkLoaderSpec2; 156 | cassandraBulkLoader2.run(); 157 | 158 | // but here, we have them! 159 | try (final Cluster cluster = Cluster.builder().addContactPoint("127.0.0.1").withPort(9042).build(); final Session session = cluster.connect()) { 160 | List results = session.execute(QueryBuilder.select().all().from("test", "test")).all(); 161 | 162 | results.forEach(row -> logger.info(format("id: %s, name: %s, surname: %s", row.getUUID("id"), row.getString("name"), row.getString("surname")))); 163 | 164 | assertEquals(3, results.size()); 165 | } 166 | } finally { 167 | if (cassandra != null) { 168 | cassandra.stop(); 169 | } 170 | } 171 | } 172 | 173 | private static Cassandra getCassandra() { 174 | CassandraBuilder builder = new CassandraBuilder(); 175 | 176 | builder.version(Version.parse(CASSANDRA_VERSION)); 177 | builder.jvmOptions("-Xmx1g"); 178 | builder.jvmOptions("-Xms1g"); 179 | builder.workingDirectory(() -> cassandraDir); 180 | // builder.addConfigProperties(new HashMap() {{ 181 | // put("enable_sasi_indexes", "true"); 182 | // put("enable_user_defined_functions", "true"); 183 | // }}); 184 | 185 | builder.workingDirectoryDestroyer(WorkingDirectoryDestroyer.deleteOnly("data")); 186 | 187 | return builder.build(); 188 | } 189 | 190 | 191 | public static final class TestFixedImplementation implements RowMapper { 192 | 193 | public static final String KEYSPACE = "test"; 194 | public static final String TABLE = "test"; 195 | 196 | public static final UUID UUID_1 = UUID.randomUUID(); 197 | public static final UUID UUID_2 = UUID.randomUUID(); 198 | public static final UUID UUID_3 = UUID.randomUUID(); 199 | 200 | @Override 201 | public List map(final List row) { 202 | return null; 203 | } 204 | 205 | @Override 206 | public Stream> get() { 207 | return Stream.of( 208 | new ArrayList() {{ 209 | add(UUID_1); 210 | add("John"); 211 | add("Doe"); 212 | }}, 213 | new ArrayList() {{ 214 | add(UUID_2); 215 | add("Marry"); 216 | add("Poppins"); 217 | }}, 218 | new ArrayList() {{ 219 | add(UUID_3); 220 | add("Jim"); 221 | add("Jack"); 222 | }}); 223 | } 224 | 225 | @Override 226 | public List random() { 227 | return null; 228 | } 229 | 230 | @Override 231 | public String insertStatement() { 232 | return format("INSERT INTO %s.%s (id, name, surname) VALUES (?, ?, ?);", KEYSPACE, TABLE); 233 | } 234 | } 235 | 236 | 237 | private void waitForCql() { 238 | await() 239 | .pollInterval(10, TimeUnit.SECONDS) 240 | .pollInSameThread() 241 | .timeout(1, TimeUnit.MINUTES) 242 | .until(() -> { 243 | try (final Cluster cluster = Cluster.builder().addContactPoint("127.0.0.1").build()) { 244 | cluster.connect(); 245 | return true; 246 | } catch (final Exception ex) { 247 | return false; 248 | } 249 | }); 250 | } 251 | 252 | public void executeWithSession(Consumer supplier) { 253 | try (final Cluster cluster = Cluster.builder().addContactPoint("127.0.0.1").build()) { 254 | try (final Session session = cluster.connect()) { 255 | supplier.accept(session); 256 | } 257 | } 258 | } 259 | 260 | private Path findCassandraYaml(final Path confDir) { 261 | 262 | try { 263 | return Files.list(confDir) 264 | .filter(path -> path.getFileName().toString().contains("-cassandra.yaml")) 265 | .findFirst() 266 | .orElseThrow(RuntimeException::new); 267 | } catch (final Exception e) { 268 | throw new IllegalStateException("Unable to list or there is not any file ending on -cassandra.yaml" + confDir); 269 | } 270 | } 271 | 272 | @Command(name = "fixed", 273 | mixinStandardHelpOptions = true, 274 | description = "tool for bulk-loading of fixed data", 275 | sortOptions = false, 276 | versionProvider = CLIApplication.class) 277 | public static final class TestBulkLoader extends BulkLoader { 278 | 279 | @Override 280 | public Generator getLoader(final BulkLoaderSpec bulkLoaderSpec, final SSTableGenerator ssTableWriter) { 281 | return new TestGenerator(ssTableWriter); 282 | } 283 | 284 | private static final class TestGenerator implements Generator { 285 | 286 | private final SSTableGenerator ssTableGenerator; 287 | 288 | public TestGenerator(final SSTableGenerator ssTableGenerator) { 289 | this.ssTableGenerator = ssTableGenerator; 290 | } 291 | 292 | @Override 293 | public void generate(final RowMapper rowMapper) { 294 | try { 295 | ssTableGenerator.generate(rowMapper.get().filter(Objects::nonNull).map(MappedRow::new).iterator()); 296 | } catch (final Exception ex) { 297 | throw new SSTableGeneratorException("Unable to generate SSTables from FixedLoader.", ex); 298 | } 299 | } 300 | } 301 | } 302 | } 303 | -------------------------------------------------------------------------------- /cassandra-4/src/test/java/com/instaclustr/cassandra/ttl/Cassandra4TTLRemoverTest.java: -------------------------------------------------------------------------------- 1 | package com.instaclustr.cassandra.ttl; 2 | 3 | import static java.lang.String.format; 4 | import static org.awaitility.Awaitility.await; 5 | import static org.junit.Assert.assertEquals; 6 | 7 | import java.io.File; 8 | import java.nio.file.Files; 9 | import java.nio.file.Path; 10 | import java.nio.file.Paths; 11 | import java.util.ArrayList; 12 | import java.util.HashMap; 13 | import java.util.List; 14 | import java.util.Objects; 15 | import java.util.UUID; 16 | import java.util.concurrent.TimeUnit; 17 | import java.util.function.Consumer; 18 | import java.util.stream.Stream; 19 | 20 | import com.datastax.driver.core.Cluster; 21 | import com.datastax.driver.core.Row; 22 | import com.datastax.driver.core.Session; 23 | import com.datastax.driver.core.querybuilder.QueryBuilder; 24 | import com.github.nosan.embedded.cassandra.Cassandra; 25 | import com.github.nosan.embedded.cassandra.CassandraBuilder; 26 | import com.github.nosan.embedded.cassandra.Version; 27 | import com.github.nosan.embedded.cassandra.WorkingDirectoryDestroyer; 28 | import com.instaclustr.cassandra.ttl.cli.TTLRemoverCLI; 29 | import com.instaclustr.sstable.generator.BulkLoader; 30 | import com.instaclustr.sstable.generator.CassandraBulkLoader; 31 | import com.instaclustr.sstable.generator.Generator; 32 | import com.instaclustr.sstable.generator.MappedRow; 33 | import com.instaclustr.sstable.generator.RowMapper; 34 | import com.instaclustr.sstable.generator.SSTableGenerator; 35 | import com.instaclustr.sstable.generator.cli.CLIApplication; 36 | import com.instaclustr.sstable.generator.exception.SSTableGeneratorException; 37 | import com.instaclustr.sstable.generator.specs.BulkLoaderSpec; 38 | import com.instaclustr.sstable.generator.specs.CassandraBulkLoaderSpec; 39 | import com.instaclustr.sstable.generator.specs.CassandraBulkLoaderSpec.CassandraVersion; 40 | import org.apache.cassandra.config.DatabaseDescriptor; 41 | import org.apache.cassandra.tools.Cassandra4CustomBulkLoader; 42 | import org.junit.Rule; 43 | import org.junit.Test; 44 | import org.junit.rules.TemporaryFolder; 45 | import org.junit.runner.RunWith; 46 | import org.junit.runners.JUnit4; 47 | import org.slf4j.Logger; 48 | import org.slf4j.LoggerFactory; 49 | import picocli.CommandLine.Command; 50 | 51 | @RunWith(JUnit4.class) 52 | public class Cassandra4TTLRemoverTest { 53 | 54 | private static final Logger logger = LoggerFactory.getLogger(Cassandra4TTLRemoverTest.class); 55 | 56 | private static final String CASSANDRA_VERSION = System.getProperty("version.cassandra4", "4.0.7"); 57 | private static final Path cassandraDir = new File("target/cassandra-4").toPath().toAbsolutePath(); 58 | 59 | private static final String KEYSPACE = "test"; 60 | private static final String TABLE = "test"; 61 | 62 | @Rule 63 | public TemporaryFolder noTTLSSTables = new TemporaryFolder(); 64 | 65 | @Rule 66 | public TemporaryFolder generatedSSTables = new TemporaryFolder(); 67 | 68 | 69 | @Test 70 | public void removeTTL() throws InterruptedException { 71 | 72 | logger.info(System.getProperty("java.library.path")); 73 | 74 | Path cassandraDir = new File("target/cassandra-4").toPath().toAbsolutePath(); 75 | 76 | Cassandra cassandra = getCassandra(); 77 | 78 | try { 79 | cassandra.start(); 80 | 81 | waitForCql(); 82 | 83 | executeWithSession(session -> { 84 | session.execute(String.format("CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1 };", KEYSPACE)); 85 | session.execute(String.format("CREATE TABLE IF NOT EXISTS %s.%s (id uuid, name text, surname text, PRIMARY KEY (id)) WITH default_time_to_live = 10;", KEYSPACE, TABLE)); 86 | }); 87 | 88 | // this has to be here for streaming in loader ... yeah, just here 89 | System.setProperty("cassandra.storagedir", cassandraDir.resolve("data").toAbsolutePath().toString()); 90 | System.setProperty("cassandra.config", "file://" + findCassandraYaml(cassandraDir.resolve("conf")).toAbsolutePath()); 91 | DatabaseDescriptor.toolInitialization(false); 92 | 93 | final BulkLoaderSpec bulkLoaderSpec = new BulkLoaderSpec(); 94 | 95 | bulkLoaderSpec.bufferSize = 128; 96 | bulkLoaderSpec.file = Paths.get(""); 97 | bulkLoaderSpec.keyspace = KEYSPACE; 98 | bulkLoaderSpec.table = TABLE; 99 | bulkLoaderSpec.partitioner = "murmur"; 100 | bulkLoaderSpec.sorted = false; 101 | bulkLoaderSpec.threads = 1; 102 | 103 | bulkLoaderSpec.generationImplementation = TestFixedImplementation.class.getName(); 104 | bulkLoaderSpec.outputDir = generatedSSTables.getRoot().toPath(); 105 | bulkLoaderSpec.schema = Paths.get(new File("src/test/resources/cassandra/cql/table.cql").getAbsolutePath()); 106 | 107 | final BulkLoader bulkLoader = new TestBulkLoader(); 108 | bulkLoader.bulkLoaderSpec = bulkLoaderSpec; 109 | 110 | bulkLoader.run(); 111 | 112 | // wait until data would expire 113 | Thread.sleep(15000); 114 | 115 | // here we see they expired 116 | 117 | try (final Cluster cluster = Cluster.builder().addContactPoint("127.0.0.1").withPort(9042).build(); final Session session = cluster.connect()) { 118 | assertEquals(0, session.execute(QueryBuilder.select().all().from("test", "test")).all().size()); 119 | } 120 | 121 | cassandra.stop(); 122 | 123 | logger.info("Removing TTLs ..."); 124 | 125 | // remove ttls 126 | 127 | TTLRemoverCLI.main(new String[]{ 128 | "--cassandra-version=4", 129 | "--sstables", 130 | bulkLoaderSpec.outputDir.toAbsolutePath() + "/test", 131 | "--output-path", 132 | noTTLSSTables.getRoot().toPath().toString(), 133 | "--cql", 134 | "CREATE TABLE IF NOT EXISTS test.test (id uuid, name text, surname text, PRIMARY KEY (id)) WITH default_time_to_live = 10;" 135 | }, false); 136 | 137 | // start new Cassandra instance 138 | 139 | cassandra.start(); 140 | 141 | executeWithSession(session -> { 142 | session.execute(String.format("CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1 };", KEYSPACE)); 143 | session.execute(String.format("CREATE TABLE IF NOT EXISTS %s.%s (id uuid, name text, surname text, PRIMARY KEY (id)) WITH default_time_to_live = 10;", KEYSPACE, TABLE)); 144 | }); 145 | 146 | final CassandraBulkLoader cassandraBulkLoader2 = new Cassandra4CustomBulkLoader(); 147 | 148 | final CassandraBulkLoaderSpec cassandraBulkLoaderSpec2 = new CassandraBulkLoaderSpec(); 149 | 150 | cassandraBulkLoaderSpec2.node = "127.0.0.1"; 151 | cassandraBulkLoaderSpec2.cassandraYaml = findCassandraYaml(cassandraDir.resolve("conf")); 152 | cassandraBulkLoaderSpec2.sstablesDir = Paths.get(noTTLSSTables.getRoot().getAbsolutePath(), KEYSPACE, TABLE); 153 | cassandraBulkLoaderSpec2.cassandraVersion = CassandraVersion.V4; 154 | cassandraBulkLoaderSpec2.keyspace = KEYSPACE; 155 | 156 | cassandraBulkLoader2.cassandraBulkLoaderSpec = cassandraBulkLoaderSpec2; 157 | cassandraBulkLoader2.run(); 158 | 159 | // but here, we have them! 160 | try (final Cluster cluster = Cluster.builder().addContactPoint("127.0.0.1").withPort(9042).build(); final Session session = cluster.connect()) { 161 | List results = session.execute(QueryBuilder.select().all().from("test", "test")).all(); 162 | 163 | results.forEach(row -> logger.info(format("id: %s, name: %s, surname: %s", row.getUUID("id"), row.getString("name"), row.getString("surname")))); 164 | 165 | assertEquals(3, results.size()); 166 | } 167 | } finally { 168 | if (cassandra != null) { 169 | cassandra.stop(); 170 | } 171 | } 172 | } 173 | 174 | private static Cassandra getCassandra() { 175 | CassandraBuilder builder = new CassandraBuilder(); 176 | 177 | builder.version(Version.parse(CASSANDRA_VERSION)); 178 | builder.jvmOptions("-Xmx1g"); 179 | builder.jvmOptions("-Xms1g"); 180 | builder.workingDirectory(() -> cassandraDir); 181 | builder.addConfigProperties(new HashMap() {{ 182 | put("enable_sasi_indexes", "true"); 183 | put("enable_user_defined_functions", "true"); 184 | }}); 185 | 186 | builder.workingDirectoryDestroyer(WorkingDirectoryDestroyer.deleteOnly("data")); 187 | 188 | return builder.build(); 189 | } 190 | 191 | 192 | public static final class TestFixedImplementation implements RowMapper { 193 | 194 | public static final String KEYSPACE = "test"; 195 | public static final String TABLE = "test"; 196 | 197 | public static final UUID UUID_1 = UUID.randomUUID(); 198 | public static final UUID UUID_2 = UUID.randomUUID(); 199 | public static final UUID UUID_3 = UUID.randomUUID(); 200 | 201 | @Override 202 | public List map(final List row) { 203 | return null; 204 | } 205 | 206 | @Override 207 | public Stream> get() { 208 | return Stream.of( 209 | new ArrayList() {{ 210 | add(UUID_1); 211 | add("John"); 212 | add("Doe"); 213 | }}, 214 | new ArrayList() {{ 215 | add(UUID_2); 216 | add("Marry"); 217 | add("Poppins"); 218 | }}, 219 | new ArrayList() {{ 220 | add(UUID_3); 221 | add("Jim"); 222 | add("Jack"); 223 | }}); 224 | } 225 | 226 | @Override 227 | public List random() { 228 | return null; 229 | } 230 | 231 | @Override 232 | public String insertStatement() { 233 | return format("INSERT INTO %s.%s (id, name, surname) VALUES (?, ?, ?);", KEYSPACE, TABLE); 234 | } 235 | } 236 | 237 | 238 | private void waitForCql() { 239 | await() 240 | .pollInterval(10, TimeUnit.SECONDS) 241 | .pollInSameThread() 242 | .timeout(1, TimeUnit.MINUTES) 243 | .until(() -> { 244 | try (final Cluster cluster = Cluster.builder().addContactPoint("127.0.0.1").build()) { 245 | cluster.connect(); 246 | return true; 247 | } catch (final Exception ex) { 248 | return false; 249 | } 250 | }); 251 | } 252 | 253 | public void executeWithSession(Consumer supplier) { 254 | try (final Cluster cluster = Cluster.builder().addContactPoint("127.0.0.1").build()) { 255 | try (final Session session = cluster.connect()) { 256 | supplier.accept(session); 257 | } 258 | } 259 | } 260 | 261 | private Path findCassandraYaml(final Path confDir) { 262 | 263 | try { 264 | return Files.list(confDir) 265 | .filter(path -> path.getFileName().toString().contains("-cassandra.yaml")) 266 | .findFirst() 267 | .orElseThrow(RuntimeException::new); 268 | } catch (final Exception e) { 269 | throw new IllegalStateException("Unable to list or there is not any file ending on -cassandra.yaml" + confDir); 270 | } 271 | } 272 | 273 | @Command(name = "fixed", 274 | mixinStandardHelpOptions = true, 275 | description = "tool for bulk-loading of fixed data", 276 | sortOptions = false, 277 | versionProvider = CLIApplication.class) 278 | public static final class TestBulkLoader extends BulkLoader { 279 | 280 | @Override 281 | public Generator getLoader(final BulkLoaderSpec bulkLoaderSpec, final SSTableGenerator ssTableWriter) { 282 | return new TestGenerator(ssTableWriter); 283 | } 284 | 285 | private static final class TestGenerator implements Generator { 286 | 287 | private final SSTableGenerator ssTableGenerator; 288 | 289 | public TestGenerator(final SSTableGenerator ssTableGenerator) { 290 | this.ssTableGenerator = ssTableGenerator; 291 | } 292 | 293 | @Override 294 | public void generate(final RowMapper rowMapper) { 295 | try { 296 | ssTableGenerator.generate(rowMapper.get().filter(Objects::nonNull).map(MappedRow::new).iterator()); 297 | } catch (final Exception ex) { 298 | throw new SSTableGeneratorException("Unable to generate SSTables from FixedLoader.", ex); 299 | } 300 | } 301 | } 302 | } 303 | } 304 | -------------------------------------------------------------------------------- /cassandra-3/src/test/java/org/apache/cassandra/ttl/Cassandra3TTLRemoverTest.java: -------------------------------------------------------------------------------- 1 | package org.apache.cassandra.ttl; 2 | 3 | import static java.lang.String.format; 4 | import static org.awaitility.Awaitility.await; 5 | import static org.junit.Assert.assertEquals; 6 | 7 | import java.io.File; 8 | import java.nio.file.Files; 9 | import java.nio.file.Path; 10 | import java.nio.file.Paths; 11 | import java.util.ArrayList; 12 | import java.util.List; 13 | import java.util.Objects; 14 | import java.util.UUID; 15 | import java.util.concurrent.TimeUnit; 16 | import java.util.function.Consumer; 17 | import java.util.stream.Stream; 18 | 19 | import com.datastax.driver.core.Cluster; 20 | import com.datastax.driver.core.Row; 21 | import com.datastax.driver.core.Session; 22 | import com.datastax.driver.core.querybuilder.QueryBuilder; 23 | import com.github.nosan.embedded.cassandra.EmbeddedCassandraFactory; 24 | import com.github.nosan.embedded.cassandra.api.Cassandra; 25 | import com.github.nosan.embedded.cassandra.api.Version; 26 | import com.github.nosan.embedded.cassandra.artifact.Artifact; 27 | import com.instaclustr.cassandra.ttl.cli.TTLRemoverCLI; 28 | import com.instaclustr.sstable.generator.BulkLoader; 29 | import com.instaclustr.sstable.generator.CassandraBulkLoader; 30 | import com.instaclustr.sstable.generator.Generator; 31 | import com.instaclustr.sstable.generator.MappedRow; 32 | import com.instaclustr.sstable.generator.RowMapper; 33 | import com.instaclustr.sstable.generator.SSTableGenerator; 34 | import com.instaclustr.sstable.generator.cli.CLIApplication; 35 | import com.instaclustr.sstable.generator.exception.SSTableGeneratorException; 36 | import com.instaclustr.sstable.generator.specs.BulkLoaderSpec; 37 | import com.instaclustr.sstable.generator.specs.CassandraBulkLoaderSpec; 38 | import com.instaclustr.sstable.generator.specs.CassandraBulkLoaderSpec.CassandraVersion; 39 | import org.apache.cassandra.config.DatabaseDescriptor; 40 | import org.apache.cassandra.tools.Cassandra3CustomBulkLoader; 41 | import org.junit.Rule; 42 | import org.junit.Test; 43 | import org.junit.rules.TemporaryFolder; 44 | import org.junit.runner.RunWith; 45 | import org.junit.runners.JUnit4; 46 | import org.slf4j.Logger; 47 | import org.slf4j.LoggerFactory; 48 | import picocli.CommandLine.Command; 49 | 50 | @RunWith(JUnit4.class) 51 | public class Cassandra3TTLRemoverTest { 52 | 53 | private static final Logger logger = LoggerFactory.getLogger(Cassandra3TTLRemoverTest.class); 54 | 55 | private static final String CASSANDRA_VERSION = System.getProperty("version.cassandra3", "3.11.14"); 56 | 57 | private static final String KEYSPACE = "test"; 58 | 59 | private static final String TABLE = "test"; 60 | 61 | private static final Artifact CASSANDRA_ARTIFACT = Artifact.ofVersion(Version.of(CASSANDRA_VERSION)); 62 | 63 | @Rule 64 | public TemporaryFolder noTTLSSTables = new TemporaryFolder(); 65 | 66 | @Rule 67 | public TemporaryFolder generatedSSTables = new TemporaryFolder(); 68 | 69 | @Test 70 | public void removeTTL() throws InterruptedException { 71 | 72 | Path cassandraDir = new File("target/cassandra-3").toPath().toAbsolutePath(); 73 | 74 | EmbeddedCassandraFactory cassandraFactory = new EmbeddedCassandraFactory(); 75 | cassandraFactory.setWorkingDirectory(cassandraDir); 76 | cassandraFactory.setArtifact(CASSANDRA_ARTIFACT); 77 | cassandraFactory.getJvmOptions().add("-Xmx1g"); 78 | cassandraFactory.getJvmOptions().add("-Xms1g"); 79 | 80 | Cassandra cassandra = null; 81 | 82 | try { 83 | cassandra = cassandraFactory.create(); 84 | cassandra.start(); 85 | 86 | waitForCql(); 87 | 88 | executeWithSession(session -> { 89 | session.execute(format("CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1 };", KEYSPACE)); 90 | session.execute(format("CREATE TABLE IF NOT EXISTS %s.%s (id uuid, name text, surname text, PRIMARY KEY (id)) WITH default_time_to_live = 10;", KEYSPACE, TABLE)); 91 | }); 92 | 93 | // this has to be here for streaming in loader ... yeah, just here 94 | System.setProperty("cassandra.storagedir", cassandraDir.resolve("data").toAbsolutePath().toString()); 95 | System.setProperty("cassandra.config", "file://" + findCassandraYaml(new File("target/cassandra-3/conf").toPath()).toAbsolutePath()); 96 | DatabaseDescriptor.toolInitialization(false); 97 | 98 | // SSTable generation 99 | 100 | final BulkLoaderSpec bulkLoaderSpec = new BulkLoaderSpec(); 101 | 102 | bulkLoaderSpec.bufferSize = 128; 103 | bulkLoaderSpec.file = Paths.get(""); 104 | bulkLoaderSpec.keyspace = KEYSPACE; 105 | bulkLoaderSpec.table = TABLE; 106 | bulkLoaderSpec.partitioner = "murmur"; 107 | bulkLoaderSpec.sorted = false; 108 | bulkLoaderSpec.threads = 1; 109 | 110 | bulkLoaderSpec.generationImplementation = TestFixedImplementation.class.getName(); 111 | bulkLoaderSpec.outputDir = generatedSSTables.getRoot().toPath(); 112 | bulkLoaderSpec.schema = Paths.get(new File("src/test/resources/cassandra/cql/table.cql").getAbsolutePath()); 113 | 114 | final BulkLoader bulkLoader = new TestBulkLoader(); 115 | bulkLoader.bulkLoaderSpec = bulkLoaderSpec; 116 | 117 | bulkLoader.run(); 118 | 119 | // wait until data would expire 120 | Thread.sleep(15000); 121 | 122 | // 123 | // load data and see that we do not have them there as they expired 124 | 125 | final CassandraBulkLoaderSpec cassandraBulkLoaderSpec = new CassandraBulkLoaderSpec(); 126 | cassandraBulkLoaderSpec.node = "127.0.0.1"; 127 | cassandraBulkLoaderSpec.cassandraYaml = findCassandraYaml(new File("target/cassandra-3/conf").toPath()); 128 | cassandraBulkLoaderSpec.sstablesDir = bulkLoaderSpec.outputDir; 129 | cassandraBulkLoaderSpec.cassandraVersion = CassandraVersion.V3; 130 | 131 | final CassandraBulkLoader cassandraBulkLoader = new Cassandra3CustomBulkLoader(); 132 | cassandraBulkLoader.cassandraBulkLoaderSpec = cassandraBulkLoaderSpec; 133 | 134 | // here we see they expired 135 | 136 | try (final Cluster cluster = Cluster.builder().addContactPoint("127.0.0.1").withPort(9042).build(); final Session session = cluster.connect()) { 137 | assertEquals(0, session.execute(QueryBuilder.select().all().from("test", "test")).all().size()); 138 | } 139 | 140 | cassandra.stop(); 141 | 142 | // start new Cassandra instance 143 | 144 | cassandra = cassandraFactory.create(); 145 | cassandra.start(); 146 | 147 | executeWithSession(session -> { 148 | session.execute(format("CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1 };", KEYSPACE)); 149 | session.execute(format("CREATE TABLE IF NOT EXISTS %s.%s (id uuid, name text, surname text, PRIMARY KEY (id)) WITH default_time_to_live = 10;", KEYSPACE, TABLE)); 150 | }); 151 | 152 | // remove ttls 153 | 154 | logger.info("Removing TTLs ..."); 155 | 156 | TTLRemoverCLI.main(new String[]{ 157 | "--cassandra-version=3", 158 | "--sstables", 159 | bulkLoaderSpec.outputDir.toAbsolutePath() + "/test", 160 | "--output-path", 161 | noTTLSSTables.getRoot().toPath().toString(), 162 | "--cql", 163 | "CREATE TABLE IF NOT EXISTS test.test (id uuid, name text, surname text, PRIMARY KEY (id)) WITH default_time_to_live = 10;" 164 | }, false); 165 | 166 | // import it into Cassandra 167 | 168 | final CassandraBulkLoaderSpec cassandraBulkLoaderSpec2 = new CassandraBulkLoaderSpec(); 169 | 170 | cassandraBulkLoaderSpec2.node = "127.0.0.1"; 171 | cassandraBulkLoaderSpec2.cassandraYaml = findCassandraYaml(new File("target/cassandra-3/conf").toPath()); 172 | cassandraBulkLoaderSpec2.sstablesDir = Paths.get(noTTLSSTables.getRoot().getAbsolutePath(), KEYSPACE, TABLE); 173 | cassandraBulkLoaderSpec2.cassandraVersion = CassandraVersion.V3; 174 | 175 | cassandraBulkLoader.cassandraBulkLoaderSpec = cassandraBulkLoaderSpec2; 176 | cassandraBulkLoader.run(); 177 | 178 | // but here, we have them! 179 | try (final Cluster cluster = Cluster.builder().addContactPoint("127.0.0.1").withPort(9042).build(); final Session session = cluster.connect()) { 180 | 181 | List results = session.execute(QueryBuilder.select().all().from("test", "test")).all(); 182 | 183 | results.forEach(row -> logger.info(format("id: %s, name: %s, surname: %s", row.getUUID("id"), row.getString("name"), row.getString("surname")))); 184 | 185 | assertEquals(3, results.size()); 186 | } 187 | } finally { 188 | if (cassandra != null) { 189 | cassandra.stop(); 190 | } 191 | } 192 | } 193 | 194 | public static final class TestFixedImplementation implements RowMapper { 195 | 196 | public static final String KEYSPACE = "test"; 197 | public static final String TABLE = "test"; 198 | 199 | public static final UUID UUID_1 = UUID.randomUUID(); 200 | public static final UUID UUID_2 = UUID.randomUUID(); 201 | public static final UUID UUID_3 = UUID.randomUUID(); 202 | 203 | @Override 204 | public List map(final List row) { 205 | return null; 206 | } 207 | 208 | @Override 209 | public Stream> get() { 210 | return Stream.of( 211 | new ArrayList() {{ 212 | add(UUID_1); 213 | add("John"); 214 | add("Doe"); 215 | }}, 216 | new ArrayList() {{ 217 | add(UUID_2); 218 | add("Marry"); 219 | add("Poppins"); 220 | }}, 221 | new ArrayList() {{ 222 | add(UUID_3); 223 | add("Jim"); 224 | add("Jack"); 225 | }}); 226 | } 227 | 228 | @Override 229 | public List random() { 230 | return null; 231 | } 232 | 233 | @Override 234 | public String insertStatement() { 235 | return format("INSERT INTO %s.%s (id, name, surname) VALUES (?, ?, ?);", KEYSPACE, TABLE); 236 | } 237 | } 238 | 239 | 240 | private void waitForCql() { 241 | await() 242 | .pollInterval(10, TimeUnit.SECONDS) 243 | .pollInSameThread() 244 | .timeout(1, TimeUnit.MINUTES) 245 | .until(() -> { 246 | try (final Cluster cluster = Cluster.builder().addContactPoint("127.0.0.1").build()) { 247 | cluster.connect(); 248 | return true; 249 | } catch (final Exception ex) { 250 | return false; 251 | } 252 | }); 253 | } 254 | 255 | public void executeWithSession(Consumer supplier) { 256 | try (final Cluster cluster = Cluster.builder().addContactPoint("127.0.0.1").build()) { 257 | try (final Session session = cluster.connect()) { 258 | supplier.accept(session); 259 | } 260 | } 261 | } 262 | 263 | private Path findCassandraYaml(final Path confDir) { 264 | 265 | try { 266 | return Files.list(confDir) 267 | .filter(path -> path.getFileName().toString().contains("-cassandra.yaml")) 268 | .findFirst() 269 | .orElseThrow(RuntimeException::new); 270 | } catch (final Exception e) { 271 | throw new IllegalStateException("Unable to list or there is not any file ending on -cassandra.yaml" + confDir); 272 | } 273 | } 274 | 275 | @Command(name = "fixed", 276 | mixinStandardHelpOptions = true, 277 | description = "tool for bulk-loading of fixed data", 278 | sortOptions = false, 279 | versionProvider = CLIApplication.class) 280 | public static final class TestBulkLoader extends BulkLoader { 281 | 282 | @Override 283 | public Generator getLoader(final BulkLoaderSpec bulkLoaderSpec, final SSTableGenerator ssTableWriter) { 284 | return new TestGenerator(ssTableWriter); 285 | } 286 | 287 | private static final class TestGenerator implements Generator { 288 | 289 | private final SSTableGenerator ssTableGenerator; 290 | 291 | public TestGenerator(final SSTableGenerator ssTableGenerator) { 292 | this.ssTableGenerator = ssTableGenerator; 293 | } 294 | 295 | @Override 296 | public void generate(final RowMapper rowMapper) { 297 | try { 298 | ssTableGenerator.generate(rowMapper.get().filter(Objects::nonNull).map(MappedRow::new).iterator()); 299 | } catch (final Exception ex) { 300 | throw new SSTableGeneratorException("Unable to generate SSTables from FixedLoader.", ex); 301 | } 302 | } 303 | } 304 | } 305 | } 306 | -------------------------------------------------------------------------------- /cassandra-2/src/test/java/org/apache/cassandra/ttl/Cassandra2TTLRemoverTest.java: -------------------------------------------------------------------------------- 1 | package org.apache.cassandra.ttl; 2 | 3 | import static com.instaclustr.cassandra.ttl.cli.TTLRemoverCLI.CassandraVersion.V2; 4 | import static java.lang.String.format; 5 | import static org.awaitility.Awaitility.await; 6 | import static org.junit.Assert.assertEquals; 7 | 8 | import java.io.File; 9 | import java.nio.file.Files; 10 | import java.nio.file.Path; 11 | import java.nio.file.Paths; 12 | import java.util.ArrayList; 13 | import java.util.List; 14 | import java.util.Objects; 15 | import java.util.UUID; 16 | import java.util.concurrent.TimeUnit; 17 | import java.util.function.Consumer; 18 | import java.util.stream.Stream; 19 | 20 | import com.datastax.driver.core.Cluster; 21 | import com.datastax.driver.core.Session; 22 | import com.datastax.driver.core.querybuilder.QueryBuilder; 23 | import com.github.nosan.embedded.cassandra.EmbeddedCassandraFactory; 24 | import com.github.nosan.embedded.cassandra.api.Cassandra; 25 | import com.github.nosan.embedded.cassandra.api.Version; 26 | import com.github.nosan.embedded.cassandra.artifact.Artifact; 27 | import com.instaclustr.cassandra.ttl.cli.TTLRemoverCLI; 28 | import com.instaclustr.sstable.generator.BulkLoader; 29 | import com.instaclustr.sstable.generator.CassandraBulkLoader; 30 | import com.instaclustr.sstable.generator.Generator; 31 | import com.instaclustr.sstable.generator.MappedRow; 32 | import com.instaclustr.sstable.generator.RowMapper; 33 | import com.instaclustr.sstable.generator.SSTableGenerator; 34 | import com.instaclustr.sstable.generator.cli.CLIApplication; 35 | import com.instaclustr.sstable.generator.exception.SSTableGeneratorException; 36 | import com.instaclustr.sstable.generator.specs.BulkLoaderSpec; 37 | import com.instaclustr.sstable.generator.specs.CassandraBulkLoaderSpec; 38 | import com.instaclustr.sstable.generator.specs.CassandraBulkLoaderSpec.CassandraVersion; 39 | import org.apache.cassandra.config.Config; 40 | import org.apache.cassandra.tools.Cassandra2CustomBulkLoader; 41 | import org.junit.Rule; 42 | import org.junit.Test; 43 | import org.junit.rules.TemporaryFolder; 44 | import org.junit.runner.RunWith; 45 | import org.junit.runners.JUnit4; 46 | import org.slf4j.Logger; 47 | import org.slf4j.LoggerFactory; 48 | import picocli.CommandLine.Command; 49 | 50 | @RunWith(JUnit4.class) 51 | public class Cassandra2TTLRemoverTest { 52 | 53 | private static final Logger logger = LoggerFactory.getLogger(Cassandra2TTLRemoverTest.class); 54 | 55 | private static final String CASSANDRA_VERSION = System.getProperty("version.cassandra2", "2.2.19"); 56 | 57 | private static final String KEYSPACE = "test"; 58 | 59 | private static final String TABLE = "test"; 60 | 61 | private static final Artifact CASSANDRA_ARTIFACT = Artifact.ofVersion(Version.of(CASSANDRA_VERSION)); 62 | 63 | @Rule 64 | public TemporaryFolder noTTLSSTables = new TemporaryFolder(); 65 | 66 | @Rule 67 | public TemporaryFolder generatedSSTables = new TemporaryFolder(); 68 | 69 | @Test 70 | public void removeTTL() throws InterruptedException { 71 | 72 | Path cassandraDir = new File("target/cassandra-2").toPath().toAbsolutePath(); 73 | 74 | EmbeddedCassandraFactory cassandraFactory = new EmbeddedCassandraFactory(); 75 | cassandraFactory.setWorkingDirectory(cassandraDir); 76 | cassandraFactory.setArtifact(CASSANDRA_ARTIFACT); 77 | cassandraFactory.getJvmOptions().add("-Xmx1g"); 78 | cassandraFactory.getJvmOptions().add("-Xms1g"); 79 | cassandraFactory.getConfigProperties().put("data_file_directories", new String[]{cassandraDir.resolve("data").toString()}); 80 | cassandraFactory.getConfigProperties().put("file_cache_size_in_mb", "1"); 81 | 82 | Cassandra cassandra = null; 83 | 84 | try { 85 | cassandra = cassandraFactory.create(); 86 | cassandra.start(); 87 | 88 | waitForCql(); 89 | 90 | executeWithSession(session -> { 91 | session.execute(format("CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1 };", KEYSPACE)); 92 | session.execute(format("CREATE TABLE IF NOT EXISTS %s.%s (id uuid, name text, surname text, PRIMARY KEY (id)) WITH default_time_to_live = 10;", KEYSPACE, TABLE)); 93 | }); 94 | 95 | // SSTable generation 96 | 97 | Config.setClientMode(false); 98 | 99 | System.setProperty("ttl.remover.tests", "true"); 100 | 101 | TTLRemoverCLI.setProperties(findCassandraYaml(new File("target/cassandra-2/conf").toPath()), cassandraDir.resolve("data").toAbsolutePath(), V2); 102 | 103 | final BulkLoaderSpec bulkLoaderSpec = new BulkLoaderSpec(); 104 | 105 | bulkLoaderSpec.bufferSize = 128; 106 | bulkLoaderSpec.file = Paths.get(""); 107 | bulkLoaderSpec.keyspace = KEYSPACE; 108 | bulkLoaderSpec.table = TABLE; 109 | bulkLoaderSpec.partitioner = "murmur"; 110 | bulkLoaderSpec.sorted = false; 111 | bulkLoaderSpec.threads = 1; 112 | 113 | bulkLoaderSpec.generationImplementation = TestFixedImplementation.class.getName(); 114 | bulkLoaderSpec.outputDir = generatedSSTables.getRoot().toPath(); 115 | bulkLoaderSpec.schema = Paths.get(new File("src/test/resources/cassandra/cql/table.cql").getAbsolutePath()); 116 | 117 | final BulkLoader bulkLoader = new TestBulkLoader(); 118 | bulkLoader.bulkLoaderSpec = bulkLoaderSpec; 119 | 120 | bulkLoader.run(); 121 | 122 | // wait until data would expire 123 | Thread.sleep(15000); 124 | 125 | // 126 | // load data and see that we do not have them there as they expired 127 | 128 | Config.setClientMode(false); 129 | 130 | final CassandraBulkLoaderSpec cassandraBulkLoaderSpec = new CassandraBulkLoaderSpec(); 131 | cassandraBulkLoaderSpec.node = "127.0.0.1"; 132 | cassandraBulkLoaderSpec.cassandraYaml = findCassandraYaml(new File("target/cassandra-2/conf").toPath()); 133 | cassandraBulkLoaderSpec.sstablesDir = bulkLoaderSpec.outputDir; 134 | cassandraBulkLoaderSpec.cassandraVersion = CassandraVersion.V3; 135 | 136 | final CassandraBulkLoader cassandraBulkLoader = new Cassandra2CustomBulkLoader(); 137 | cassandraBulkLoader.cassandraBulkLoaderSpec = cassandraBulkLoaderSpec; 138 | 139 | // here we see they expired 140 | 141 | try (final Cluster cluster = Cluster.builder().addContactPoint("127.0.0.1").withPort(9042).build(); final Session session = cluster.connect()) { 142 | assertEquals(0, session.execute(QueryBuilder.select().all().from("test", "test")).all().size()); 143 | } 144 | 145 | cassandra.stop(); 146 | 147 | // start new Cassandra instance 148 | 149 | cassandra = cassandraFactory.create(); 150 | cassandra.start(); 151 | 152 | TTLRemoverCLI.setProperties(findCassandraYaml(new File("target/cassandra-2/conf").toPath()), cassandraDir.resolve("data").toAbsolutePath(), V2); 153 | Config.setClientMode(false); 154 | 155 | executeWithSession(session -> { 156 | session.execute(format("CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1 };", KEYSPACE)); 157 | session.execute(format("CREATE TABLE IF NOT EXISTS %s.%s (id uuid, name text, surname text, PRIMARY KEY (id)) WITH default_time_to_live = 10;", KEYSPACE, TABLE)); 158 | }); 159 | 160 | // remove ttls 161 | 162 | logger.info("Removing TTLs ..."); 163 | 164 | TTLRemoverCLI.main(new String[]{ 165 | "--cassandra-version=2", 166 | "--sstables", 167 | bulkLoaderSpec.outputDir.toAbsolutePath() + "/test", 168 | "--output-path", 169 | noTTLSSTables.getRoot().toPath().toString(), 170 | "--cassandra-yaml", 171 | findCassandraYaml(new File("target/cassandra-2/conf").toPath()).toAbsolutePath().toString(), 172 | "--cassandra-storage-dir", 173 | new File("target/cassandra-2/data/data").getAbsolutePath(), 174 | }, false); 175 | 176 | // import it into Cassandra 177 | 178 | final CassandraBulkLoaderSpec cassandraBulkLoaderSpec2 = new CassandraBulkLoaderSpec(); 179 | 180 | cassandraBulkLoaderSpec2.node = "127.0.0.1"; 181 | cassandraBulkLoaderSpec2.cassandraYaml = findCassandraYaml(new File("target/cassandra-2/conf").toPath()); 182 | cassandraBulkLoaderSpec2.sstablesDir = Paths.get(noTTLSSTables.getRoot().getAbsolutePath(), KEYSPACE, TABLE); 183 | cassandraBulkLoaderSpec2.cassandraVersion = CassandraVersion.V3; 184 | 185 | cassandraBulkLoader.cassandraBulkLoaderSpec = cassandraBulkLoaderSpec2; 186 | cassandraBulkLoader.run(); 187 | 188 | // but here, we have them! 189 | try (final Cluster cluster = Cluster.builder().addContactPoint("127.0.0.1").withPort(9042).build(); final Session session = cluster.connect()) { 190 | assertEquals(3, session.execute(QueryBuilder.select().all().from("test", "test")).all().size()); 191 | } 192 | } finally { 193 | if (cassandra != null) { 194 | cassandra.stop(); 195 | } 196 | } 197 | } 198 | 199 | public static final class TestFixedImplementation implements RowMapper { 200 | 201 | public static final String KEYSPACE = "test"; 202 | public static final String TABLE = "test"; 203 | 204 | public static final UUID UUID_1 = UUID.randomUUID(); 205 | public static final UUID UUID_2 = UUID.randomUUID(); 206 | public static final UUID UUID_3 = UUID.randomUUID(); 207 | 208 | @Override 209 | public List map(final List row) { 210 | return null; 211 | } 212 | 213 | @Override 214 | public Stream> get() { 215 | return Stream.of( 216 | new ArrayList() {{ 217 | add(UUID_1); 218 | add("John"); 219 | add("Doe"); 220 | }}, 221 | new ArrayList() {{ 222 | add(UUID_2); 223 | add("Marry"); 224 | add("Poppins"); 225 | }}, 226 | new ArrayList() {{ 227 | add(UUID_3); 228 | add("Jim"); 229 | add("Jack"); 230 | }}); 231 | } 232 | 233 | @Override 234 | public List random() { 235 | return null; 236 | } 237 | 238 | @Override 239 | public String insertStatement() { 240 | return format("INSERT INTO %s.%s (id, name, surname) VALUES (?, ?, ?);", KEYSPACE, TABLE); 241 | } 242 | } 243 | 244 | 245 | private void waitForCql() { 246 | await() 247 | .pollInterval(10, TimeUnit.SECONDS) 248 | .pollInSameThread() 249 | .timeout(1, TimeUnit.MINUTES) 250 | .until(() -> { 251 | try (final Cluster cluster = Cluster.builder().addContactPoint("127.0.0.1").build()) { 252 | cluster.connect(); 253 | return true; 254 | } catch (final Exception ex) { 255 | return false; 256 | } 257 | }); 258 | } 259 | 260 | public void executeWithSession(Consumer supplier) { 261 | try (final Cluster cluster = Cluster.builder().addContactPoint("127.0.0.1").build()) { 262 | try (final Session session = cluster.connect()) { 263 | supplier.accept(session); 264 | } 265 | } 266 | } 267 | 268 | private Path findCassandraYaml(final Path confDir) { 269 | 270 | try { 271 | return Files.list(confDir) 272 | .filter(path -> path.getFileName().toString().contains("-cassandra.yaml")) 273 | .findFirst() 274 | .orElseThrow(RuntimeException::new); 275 | } catch (final Exception e) { 276 | throw new IllegalStateException("Unable to list or there is not any file ending on -cassandra.yaml" + confDir); 277 | } 278 | } 279 | 280 | @Command(name = "fixed", 281 | mixinStandardHelpOptions = true, 282 | description = "tool for bulk-loading of fixed data", 283 | sortOptions = false, 284 | versionProvider = CLIApplication.class) 285 | public static final class TestBulkLoader extends BulkLoader { 286 | 287 | @Override 288 | public Generator getLoader(final BulkLoaderSpec bulkLoaderSpec, final SSTableGenerator ssTableWriter) { 289 | return new TestGenerator(ssTableWriter); 290 | } 291 | 292 | private static final class TestGenerator implements Generator { 293 | 294 | private final SSTableGenerator ssTableGenerator; 295 | 296 | public TestGenerator(final SSTableGenerator ssTableGenerator) { 297 | this.ssTableGenerator = ssTableGenerator; 298 | } 299 | 300 | @Override 301 | public void generate(final RowMapper rowMapper) { 302 | try { 303 | ssTableGenerator.generate(rowMapper.get().filter(Objects::nonNull).map(MappedRow::new).iterator()); 304 | } catch (final Exception ex) { 305 | throw new SSTableGeneratorException("Unable to generate SSTables from FixedLoader.", ex); 306 | } 307 | } 308 | } 309 | } 310 | } 311 | --------------------------------------------------------------------------------