├── .gitignore ├── .travis.yml ├── DISCLAIMER ├── KEYS ├── LICENSE.txt ├── NOTICE ├── README.md ├── benchmarks ├── bin │ ├── benchmarks.sh │ └── omid-env.sh ├── conf │ ├── hbase-site.xml │ └── log4j.xml ├── maven │ └── assembly │ │ └── bin.xml ├── pom.xml └── src │ └── main │ ├── java │ └── org │ │ └── apache │ │ └── omid │ │ └── benchmarks │ │ ├── hbase │ │ └── HBaseCommitTableTester.java │ │ ├── tso │ │ ├── RawTxRunner.java │ │ ├── TSOServerBenchmark.java │ │ └── TSOServerBenchmarkConfig.java │ │ └── utils │ │ ├── Generator.java │ │ ├── IntegerGenerator.java │ │ ├── ScrambledZipfianGenerator.java │ │ ├── UniformGenerator.java │ │ └── ZipfianGenerator.java │ └── resources │ └── default-tso-server-benchmark-config.yml ├── bintray-settings.xml ├── codahale-metrics ├── pom.xml └── src │ └── main │ └── java │ └── org │ └── apache │ └── omid │ └── metrics │ ├── CodahaleMetricsConfig.java │ └── CodahaleMetricsProvider.java ├── commit-table ├── pom.xml └── src │ ├── main │ └── java │ │ └── org │ │ └── apache │ │ └── omid │ │ └── committable │ │ ├── CommitTable.java │ │ ├── InMemoryCommitTable.java │ │ └── NullCommitTable.java │ └── test │ └── java │ └── org │ └── apache │ └── omid │ └── committable │ └── NullCommitTableTest.java ├── common ├── pom.xml └── src │ ├── main │ ├── java │ │ └── org │ │ │ └── apache │ │ │ └── omid │ │ │ ├── NetworkUtils.java │ │ │ ├── YAMLUtils.java │ │ │ └── zk │ │ │ └── ZKUtils.java │ └── proto │ │ └── TSOProto.proto │ └── test │ ├── java │ └── org │ │ └── apache │ │ └── omid │ │ └── YAMLUtilsTest.java │ └── resources │ ├── default-test.yml │ └── test.yml ├── doc ├── images │ ├── ModuleDependencies.graffle │ ├── ModuleDependencies.png │ └── omid-logo.png └── site │ ├── markdown │ ├── basic-algorithm.md │ ├── basic-concepts.md │ ├── basic-examples.md │ ├── client-failure-management.md │ ├── coding-guide-and-style.md │ ├── index.md │ ├── mailing-lists.md │ ├── omid-components.md │ └── quickstart.md │ ├── resources │ ├── css │ │ └── site.css │ └── images │ │ ├── about-omid.png │ │ ├── architecture.png │ │ ├── basic-alg.png │ │ ├── contact.png │ │ ├── getting-started.png │ │ ├── omid-logo-transparent.png │ │ ├── omid-logo.png │ │ ├── snapshot-isolation.png │ │ └── technical-documentation.png │ └── site.xml ├── examples ├── maven │ └── assembly │ │ └── assembly.xml ├── pom.xml ├── run.sh └── src │ └── main │ ├── java │ └── org │ │ └── apache │ │ └── omid │ │ └── examples │ │ ├── BasicExample.java │ │ ├── ConfigurationExample.java │ │ ├── ParallelExecution.java │ │ ├── RowIdGenerator.java │ │ └── SnapshotIsolationExample.java │ └── resources │ ├── hbase-omid-client-config.yml │ └── log4j.properties ├── hbase-client ├── pom.xml └── src │ ├── main │ ├── java │ │ └── org │ │ │ └── apache │ │ │ └── omid │ │ │ └── transaction │ │ │ ├── ColumnWrapper.java │ │ │ ├── HBaseAsyncPostCommitter.java │ │ │ ├── HBaseCellId.java │ │ │ ├── HBaseOmidClientConfiguration.java │ │ │ ├── HBaseSyncPostCommitter.java │ │ │ ├── HBaseTransaction.java │ │ │ ├── HBaseTransactionClient.java │ │ │ ├── HBaseTransactionManager.java │ │ │ ├── TTable.java │ │ │ └── TTableCellGetterAdapter.java │ └── resources │ │ └── default-hbase-omid-client-config.yml │ └── test │ ├── java │ └── org │ │ └── apache │ │ └── omid │ │ └── transaction │ │ ├── OmidTestBase.java │ │ ├── TestAsynchronousPostCommitter.java │ │ ├── TestAutoFlush.java │ │ ├── TestBaillisAnomaliesWithTXs.java │ │ ├── TestBasicTransaction.java │ │ ├── TestCellUtils.java │ │ ├── TestColumnIterator.java │ │ ├── TestDeletion.java │ │ ├── TestEndToEndScenariosWithHA.java │ │ ├── TestFilters.java │ │ ├── TestHALeaseManagementModule.java │ │ ├── TestHBaseOmidClientConfiguration.java │ │ ├── TestHBaseTransactionClient.java │ │ ├── TestHBaseTransactionManager.java │ │ ├── TestMultiplePut.java │ │ ├── TestReadPath.java │ │ ├── TestShadowCells.java │ │ ├── TestSingleColumnFamily.java │ │ ├── TestTSOModule.java │ │ ├── TestTTableBehaviour.java │ │ ├── TestTransactionCleanup.java │ │ ├── TestTransactionConflict.java │ │ ├── TestTxMgrFailover.java │ │ └── TestUpdateScan.java │ └── resources │ ├── log4j.properties │ └── test-hbase-omid-client-config.yml ├── hbase-commit-table ├── pom.xml └── src │ ├── main │ └── java │ │ └── org │ │ └── apache │ │ └── omid │ │ └── committable │ │ └── hbase │ │ ├── DefaultHBaseCommitTableStorageModule.java │ │ ├── HBaseCommitTable.java │ │ ├── HBaseCommitTableConfig.java │ │ └── HBaseCommitTableStorageModule.java │ └── test │ └── java │ └── org │ └── apache │ └── omid │ └── committable │ └── hbase │ └── TestHBaseCommitTable.java ├── hbase-common ├── pom.xml └── src │ ├── main │ └── java │ │ └── org │ │ └── apache │ │ └── omid │ │ ├── HBaseConfigModule.java │ │ ├── committable │ │ └── hbase │ │ │ ├── KeyGenerator.java │ │ │ ├── KeyGeneratorImplementations.java │ │ │ └── RegionSplitter.java │ │ ├── tools │ │ └── hbase │ │ │ ├── HBaseLogin.java │ │ │ └── SecureHBaseConfig.java │ │ └── transaction │ │ ├── CellInfo.java │ │ └── CellUtils.java │ └── test │ └── java │ └── org │ └── apache │ └── omid │ └── committable │ └── hbase │ └── TestHBaseCommitTableKey.java ├── hbase-coprocessor ├── .gitignore ├── pom.xml └── src │ ├── main │ └── java │ │ └── org │ │ └── apache │ │ ├── hadoop │ │ └── hbase │ │ │ └── regionserver │ │ │ └── CompactorScanner.java │ │ └── omid │ │ └── transaction │ │ ├── CompactorUtil.java │ │ ├── HRegionCellGetterAdapter.java │ │ └── OmidCompactor.java │ └── test │ ├── java │ └── org │ │ └── apache │ │ └── omid │ │ └── transaction │ │ ├── TSOForHBaseCompactorTestModule.java │ │ ├── TestCompaction.java │ │ └── TestCompactorScanner.java │ └── resources │ └── log4j.xml ├── hbase-shims ├── hbase-0 │ ├── pom.xml │ └── src │ │ └── main │ │ └── java │ │ └── org │ │ └── apache │ │ ├── hadoop │ │ └── hbase │ │ │ └── regionserver │ │ │ ├── Region.java │ │ │ └── ScannerContext.java │ │ └── omid │ │ └── HBaseShims.java ├── hbase-1 │ ├── pom.xml │ └── src │ │ └── main │ │ └── java │ │ └── org │ │ └── apache │ │ └── omid │ │ └── HBaseShims.java └── pom.xml ├── hbase-tools ├── pom.xml └── src │ ├── main │ └── java │ │ └── org │ │ └── apache │ │ └── omid │ │ └── tools │ │ └── hbase │ │ └── OmidTableManager.java │ └── test │ └── java │ └── org │ └── apache │ └── omid │ └── tools │ └── hbase │ └── TestOmidTableManager.java ├── metrics ├── pom.xml └── src │ └── main │ └── java │ └── org │ └── apache │ └── omid │ └── metrics │ ├── AbstractMetricsConfig.java │ ├── Counter.java │ ├── Gauge.java │ ├── Histogram.java │ ├── Meter.java │ ├── Metric.java │ ├── MetricsProvider.java │ ├── MetricsRegistry.java │ ├── MetricsRegistryMap.java │ ├── MetricsUtils.java │ ├── NullMetricsProvider.java │ └── Timer.java ├── misc ├── findbugs-exclude.xml ├── header.txt └── omid_checks.xml ├── packaging ├── maven │ └── assembly │ │ └── src.xml └── pom.xml ├── pom.xml ├── statemachine ├── pom.xml └── src │ ├── main │ └── java │ │ └── org │ │ └── apache │ │ └── statemachine │ │ ├── StateMachine.java │ │ └── StateMachineLogParser.java │ └── test │ └── java │ └── org │ └── apache │ └── statemachine │ └── TestStateMachine.java ├── timestamp-storage ├── pom.xml └── src │ ├── main │ └── java │ │ └── org │ │ └── apache │ │ └── omid │ │ └── timestamp │ │ └── storage │ │ ├── DefaultHBaseTimestampStorageModule.java │ │ ├── DefaultZKTimestampStorageModule.java │ │ ├── HBaseTimestampStorage.java │ │ ├── HBaseTimestampStorageConfig.java │ │ ├── HBaseTimestampStorageModule.java │ │ ├── TimestampStorage.java │ │ ├── ZKModule.java │ │ ├── ZKTimestampPaths.java │ │ ├── ZKTimestampStorage.java │ │ └── ZKTimestampStorageModule.java │ └── test │ ├── java │ └── org │ │ └── apache │ │ └── omid │ │ └── timestamp │ │ └── storage │ │ ├── TestHBaseTimestampStorage.java │ │ └── TestZKTimestampStorage.java │ └── resources │ └── log4j.properties ├── transaction-client ├── pom.xml └── src │ ├── main │ ├── java │ │ └── org │ │ │ └── apache │ │ │ └── omid │ │ │ ├── transaction │ │ │ ├── AbstractTransaction.java │ │ │ ├── AbstractTransactionManager.java │ │ │ ├── CommitTimestampLocator.java │ │ │ ├── PostCommitActions.java │ │ │ ├── RollbackException.java │ │ │ ├── Transaction.java │ │ │ ├── TransactionException.java │ │ │ ├── TransactionManager.java │ │ │ └── TransactionManagerException.java │ │ │ └── tso │ │ │ ├── client │ │ │ ├── AbortException.java │ │ │ ├── CellId.java │ │ │ ├── ClosingException.java │ │ │ ├── ConnectionException.java │ │ │ ├── ForwardingTSOFuture.java │ │ │ ├── HandshakeFailedException.java │ │ │ ├── MockTSOClient.java │ │ │ ├── OmidClientConfiguration.java │ │ │ ├── ServiceUnavailableException.java │ │ │ ├── TSOClient.java │ │ │ ├── TSOFuture.java │ │ │ └── TSOProtocol.java │ │ │ └── util │ │ │ └── DummyCellIdImpl.java │ └── resources │ │ └── omid-client-config.yml │ └── test │ └── java │ └── org │ └── apache │ └── omid │ └── tso │ └── client │ ├── TestMockTSOClient.java │ └── TestOmidClientConfiguration.java └── tso-server ├── bin ├── omid-env.sh └── omid.sh ├── conf ├── hbase-site.xml ├── log4j.xml └── omid-server-configuration.yml ├── maven └── assembly │ └── bin.xml ├── pom.xml └── src ├── main ├── java │ └── org │ │ └── apache │ │ └── omid │ │ └── tso │ │ ├── Batch.java │ │ ├── BatchPoolModule.java │ │ ├── CacheEvaluation.java │ │ ├── CommitHashMap.java │ │ ├── DisruptorModule.java │ │ ├── FatalExceptionHandler.java │ │ ├── HALeaseManagementModule.java │ │ ├── InMemoryCommitTableStorageModule.java │ │ ├── InMemoryTimestampStorageModule.java │ │ ├── LeaseManagement.java │ │ ├── LeaseManager.java │ │ ├── LongCache.java │ │ ├── MockPanicker.java │ │ ├── MonitoringContext.java │ │ ├── NetworkInterfaceUtils.java │ │ ├── Panicker.java │ │ ├── PersistEvent.java │ │ ├── PersistenceProcessor.java │ │ ├── PersistenceProcessorHandler.java │ │ ├── PersistenceProcessorImpl.java │ │ ├── ReplyProcessor.java │ │ ├── ReplyProcessorImpl.java │ │ ├── RequestProcessor.java │ │ ├── RequestProcessorImpl.java │ │ ├── RetryProcessor.java │ │ ├── RetryProcessorImpl.java │ │ ├── RuntimeExceptionPanicker.java │ │ ├── SystemExitPanicker.java │ │ ├── TSOChannelHandler.java │ │ ├── TSOModule.java │ │ ├── TSOServer.java │ │ ├── TSOServerConfig.java │ │ ├── TSOStateManager.java │ │ ├── TSOStateManagerImpl.java │ │ ├── TimestampOracle.java │ │ ├── TimestampOracleImpl.java │ │ ├── TsoServerDaemon.java │ │ ├── VoidLeaseManagementModule.java │ │ └── VoidLeaseManager.java └── resources │ ├── default-omid-server-configuration.yml │ └── log4j.properties └── test ├── java └── org │ └── apache │ └── omid │ ├── TestUtils.java │ └── tso │ ├── PausableLeaseManager.java │ ├── PausableTimestampOracle.java │ ├── ProgrammableTSOServer.java │ ├── TSOMockModule.java │ ├── TSOServerConfigTest.java │ ├── TestBatch.java │ ├── TestLeaseManager.java │ ├── TestLongCache.java │ ├── TestPanicker.java │ ├── TestPersistenceProcessor.java │ ├── TestPersistenceProcessorHandler.java │ ├── TestReplyProcessor.java │ ├── TestRequestProcessor.java │ ├── TestRetryProcessor.java │ ├── TestTSOChannelHandlerNetty.java │ ├── TestTSOStateManager.java │ ├── TestTimestampOracle.java │ └── client │ ├── TSOClientAccessor.java │ ├── TSOClientOneShot.java │ ├── TSOClientRaw.java │ ├── TestIntegrationOfTSOClientServerBasicFunctionality.java │ ├── TestTSOClientConnectionToTSO.java │ ├── TestTSOClientRequestAndResponseBehaviours.java │ ├── TestTSOClientResponseHandling.java │ └── TestUnconnectedTSOClient.java └── resources ├── log4j.properties └── test-omid.yml /.gitignore: -------------------------------------------------------------------------------- 1 | .classpath 2 | .project 3 | .settings 4 | .metadata 5 | dependency-reduced-pom.xml 6 | target/ 7 | lib/ 8 | *.log 9 | *.log.* 10 | *.orig 11 | *.iml 12 | *.ipr 13 | .idea/ 14 | *.iws 15 | *~ 16 | *.swp 17 | 18 | # Generated website files 19 | generated-website/ 20 | generated-website-staging/ 21 | 22 | # TestNG 23 | */test-output/ 24 | 25 | # Protobuf generated classes 26 | common/src/main/java/com/yahoo/omid/proto/ 27 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | 3 | notifications: 4 | email: 5 | recepients: 6 | - dev@omid.incubator.apache.org 7 | on_success: always 8 | on_failure: always 9 | 10 | jdk: 11 | - oraclejdk8 12 | 13 | branches: 14 | only: 15 | - master 16 | 17 | cache: 18 | directories: 19 | - "~/.m2" 20 | 21 | install: true 22 | 23 | before_script: 24 | # This is required to avoid failures of HBase minicluster related to Hadoop 1.x releases 25 | - umask 022 26 | - git config --global user.email "dev@omid.incubator.apache.org" 27 | - git config --global user.name "Omid CI" 28 | # Install protobuf to genearte TSO client-server protocol in each compilation 29 | - cd .. 30 | - wget https://protobuf.googlecode.com/files/protobuf-2.5.0.tar.gz 31 | - tar -xzvf protobuf-2.5.0.tar.gz 32 | - cd protobuf-2.5.0 && ./configure --prefix=/usr && make && sudo make install 33 | - cd ../incubator-omid 34 | 35 | script: 36 | - if [ "${TRAVIS_PULL_REQUEST}" = "false" ] ; then 37 | git checkout master 38 | && 39 | mvn clean cobertura:cobertura coveralls:report -Phbase-0 ; 40 | else 41 | git checkout -b tmp-build-branch 42 | && 43 | mvn clean test -Phbase-0 ; 44 | fi -------------------------------------------------------------------------------- /DISCLAIMER: -------------------------------------------------------------------------------- 1 | Apache Omid is an effort undergoing incubation at the Apache Software 2 | Foundation (ASF), sponsored by the Apache Incubator PMC. 3 | 4 | Incubation is required of all newly accepted projects until a further review 5 | indicates that the infrastructure, communications, and decision making process 6 | have stabilized in a manner consistent with other successful ASF projects. 7 | 8 | While incubation status is not necessarily a reflection of the completeness 9 | or stability of the code, it does indicate that the project has yet to be 10 | fully endorsed by the ASF. 11 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Apache Omid 2 | Copyright 2016 The Apache Software Foundation. 3 | 4 | This product includes software developed at The Apache Software Foundation (http://www.apache.org/). 5 | 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | [ ![Build Status](https://travis-ci.org/apache/incubator-omid.svg?branch=master) ](https://travis-ci.org/apache/incubator-omid) 4 | [ ![Coverage Status](https://coveralls.io/repos/github/apache/incubator-omid/badge.svg?branch=master) ](https://coveralls.io/github/apache/incubator-omid?branch=master) 5 | 6 | The Omid project provides transactional support for HBase-based applications. 7 | 8 | # Quickstart 9 | 10 | Check the Omid [basic concepts](http://omid.incubator.apache.org/) or go directly 11 | to the [quick start guide](http://omid.incubator.apache.org/quickstart.html). 12 | 13 | For a more detailed view of the system, check the sections in the technical documentation [here](http://omid.incubator.apache.org/). 14 | 15 | # License 16 | 17 | Code licensed under the Apache License Version 2.0. See LICENSE file for terms. 18 | 19 | ----------------------------------------------------------------------------------------------------------------------- 20 | 21 | Omid logo by Ignacio Prieto 22 | -------------------------------------------------------------------------------- /benchmarks/bin/benchmarks.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | 21 | SCRIPTDIR=`dirname $0` 22 | cd $SCRIPTDIR; 23 | CLASSPATH=../conf 24 | 25 | . ./omid-env.sh 26 | 27 | # for source release 28 | for j in ../target/omid-benchmarks*.jar; do 29 | CLASSPATH=$CLASSPATH:$j 30 | done 31 | 32 | # for binary release 33 | for j in ../omid-benchmarks*.jar; do 34 | CLASSPATH=$CLASSPATH:$j 35 | done 36 | for j in ../lib/*.jar; do 37 | CLASSPATH=$CLASSPATH:$j 38 | done 39 | 40 | tso() { 41 | exec java $JVM_FLAGS -Dlog4j.configuration=file:../conf/log4j.xml -cp $CLASSPATH org.apache.omid.benchmarks.tso.TSOServerBenchmark $@ 42 | } 43 | 44 | usage() { 45 | echo "Usage: benchmarks.sh " 46 | echo "where is one of:" 47 | echo " tso Starts the tso benchmark." 48 | } 49 | 50 | # if no args specified, show usage 51 | if [ $# = 0 ]; then 52 | usage; 53 | exit 1 54 | fi 55 | 56 | COMMAND=$1 57 | shift 58 | 59 | if [ "$COMMAND" = "tso" ]; then 60 | tso $@; 61 | else 62 | exec java -cp $CLASSPATH $COMMAND $@ 63 | fi 64 | 65 | 66 | -------------------------------------------------------------------------------- /benchmarks/bin/omid-env.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | # Set the flags to pass to the jvm when running omid 20 | # export JVM_FLAGS=-Xmx2048m 21 | # --------------------------------------------------------------------------------------------------------------------- 22 | # Check if HADOOP_CONF_DIR and HBASE_CONF_DIR are set 23 | # --------------------------------------------------------------------------------------------------------------------- 24 | 25 | if [ -z ${HADOOP_CONF_DIR+x} ]; then echo "WARNING: HADOOP_CONF_DIR is unset"; else echo "HADOOP_CONF_DIR is set to '$HADOOP_CONF_DIR'"; fi 26 | if [ -z ${HBASE_CONF_DIR+x} ]; then echo "WARNING: HBASE_CONF_DIR is unset"; else echo "HBASE_CONF_DIR is set to '$HBASE_CONF_DIR'"; fi 27 | 28 | -------------------------------------------------------------------------------- /benchmarks/conf/hbase-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | hbase.zookeeper.quorum 5 | localhost 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /benchmarks/conf/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /benchmarks/maven/assembly/bin.xml: -------------------------------------------------------------------------------- 1 | 4 | 5 | bin 6 | 7 | 8 | tar.gz 9 | 10 | 11 | true 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | /lib 20 | false 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | ${basedir}/*.sh 30 | 31 | 32 | 33 | bin 34 | 35 | 36 | conf 37 | 38 | 39 | 40 | 41 | 42 | 43 | target/${project.artifactId}-${project.version}.jar 44 | / 45 | omid-benchmarks.jar 46 | 47 | 48 | ${basedir}/src/main/resources/default-tso-server-benchmark-config.yml 49 | /conf 50 | tso-server-benchmark-config.yml 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | /lib 59 | false 60 | runtime 61 | false 62 | 63 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /benchmarks/src/main/java/org/apache/omid/benchmarks/utils/Generator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * http://www.apache.org/licenses/LICENSE-2.0 5 | * 6 | * Unless required by applicable law or agreed to in writing, software 7 | * distributed under the License is distributed on an "AS IS" BASIS, 8 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | * See the License for the specific language governing permissions and 10 | * limitations under the License. 11 | */ 12 | package org.apache.omid.benchmarks.utils; 13 | 14 | /** 15 | * An expression that generates a sequence of string values, following some distribution (Uniform, Zipfian, Sequential, etc.) 16 | */ 17 | public abstract class Generator { 18 | /** 19 | * Generate the next string in the distribution 20 | * @return next String 21 | */ 22 | public abstract String nextString(); 23 | 24 | /** 25 | * Return the previous string generated by the distribution; e.g., returned from the last nextString() call. 26 | * Calling lastString() should not advance the distribution or have any side effects. If nextString() has not yet 27 | * been called, lastString() should return something reasonable 28 | * @return last string 29 | */ 30 | public abstract String lastString(); 31 | } 32 | 33 | -------------------------------------------------------------------------------- /benchmarks/src/main/java/org/apache/omid/benchmarks/utils/IntegerGenerator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * http://www.apache.org/licenses/LICENSE-2.0 5 | * 6 | * Unless required by applicable law or agreed to in writing, software 7 | * distributed under the License is distributed on an "AS IS" BASIS, 8 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | * See the License for the specific language governing permissions and 10 | * limitations under the License. 11 | */ 12 | package org.apache.omid.benchmarks.utils; 13 | 14 | /** 15 | * A generator that is capable of generating ints as well as strings 16 | * 17 | * @author cooperb 18 | * 19 | */ 20 | public abstract class IntegerGenerator extends Generator { 21 | int lastint; 22 | 23 | /** 24 | * Set the last value generated. IntegerGenerator subclasses must use this call 25 | * to properly set the last string value, or the lastString() and lastInt() calls won't work. 26 | * @param last value 27 | */ 28 | protected void setLastInt(int last) { 29 | lastint = last; 30 | } 31 | 32 | /** 33 | * When overriding this method, be sure to call setLastString() properly, or the lastString() call won't work 34 | * @return Return the next value as an int. 35 | */ 36 | public abstract int nextInt(); 37 | 38 | /** 39 | * @return Generate the next string in the distribution. 40 | */ 41 | public String nextString() { 42 | return "" + nextInt(); 43 | } 44 | 45 | /** 46 | * @return the previous string generated by the distribution; e.g., returned from the last nextString() call. 47 | * Calling lastString() should not advance the distribution or have any side effects. If nextString() has not yet 48 | * been called, lastString() should return something reasonable. 49 | */ 50 | @Override 51 | public String lastString() { 52 | return "" + lastInt(); 53 | } 54 | 55 | /** 56 | * @return the previous int generated by the distribution. This call is unique to IntegerGenerator subclasses, and assumes 57 | * IntegerGenerator subclasses always return ints for nextInt() (e.g. not arbitrary strings). 58 | */ 59 | public int lastInt() { 60 | return lastint; 61 | } 62 | 63 | /** 64 | * @return the expected value (mean) of the values this generator will return. 65 | */ 66 | public abstract double mean(); 67 | } 68 | -------------------------------------------------------------------------------- /benchmarks/src/main/java/org/apache/omid/benchmarks/utils/UniformGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.benchmarks.utils; 19 | 20 | import java.util.Random; 21 | 22 | public class UniformGenerator extends IntegerGenerator { 23 | 24 | private final Random ramdom = new Random(System.nanoTime()); 25 | 26 | @Override 27 | public int nextInt() { 28 | return ramdom.nextInt(Integer.MAX_VALUE); 29 | } 30 | 31 | @Override 32 | public double mean() { 33 | return 0; 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /bintray-settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | bintray 5 | ${env.BINTRAY_USER} 6 | ${env.BINTRAY_API_KEY} 7 | 8 | 9 | -------------------------------------------------------------------------------- /codahale-metrics/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | omid 6 | org.apache.omid 7 | 0.8.2.11-SNAPSHOT 8 | 9 | 10 | 4.0.0 11 | 12 | omid-codahale-metrics 13 | Codahale Metrics 14 | 15 | 16 | 17 | 18 | 19 | 20 | org.apache.omid 21 | omid-metrics 22 | ${project.version} 23 | 24 | 25 | 26 | 27 | commons-io 28 | commons-io 29 | ${commons-io.version} 30 | 31 | 32 | 33 | 34 | 35 | com.codahale.metrics 36 | metrics-core 37 | ${metrics.version} 38 | 39 | 40 | com.codahale.metrics 41 | metrics-graphite 42 | ${metrics.version} 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | maven-site-plugin 56 | 57 | true 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /commit-table/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | 6 | org.apache.omid 7 | omid 8 | 0.8.2.11-SNAPSHOT 9 | 10 | 11 | omid-commit-table 12 | jar 13 | Commit Table 14 | 15 | 16 | 17 | 18 | 19 | 20 | com.google.guava 21 | guava 22 | ${guava.version} 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | org.testng 31 | testng 32 | ${testng.version} 33 | test 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | org.apache.maven.plugins 47 | maven-jar-plugin 48 | ${maven-jar-plugin.version} 49 | 50 | 51 | 52 | test-jar 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | maven-site-plugin 61 | 62 | true 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | -------------------------------------------------------------------------------- /commit-table/src/test/java/org/apache/omid/committable/NullCommitTableTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.committable; 19 | 20 | import org.testng.annotations.Test; 21 | 22 | import static org.testng.Assert.assertNull; 23 | 24 | /** 25 | * TODO: Remove this class when removing this class from production code 26 | */ 27 | public class NullCommitTableTest { 28 | 29 | private static final long TEST_ST = 1L; 30 | private static final long TEST_CT = 2L; 31 | private static final long TEST_LWM = 1L; 32 | 33 | @Test(timeOut = 10_000) 34 | public void testClientAndWriter() throws Exception { 35 | 36 | CommitTable commitTable = new NullCommitTable(); 37 | 38 | try (CommitTable.Client commitTableClient = commitTable.getClient(); 39 | CommitTable.Writer commitTableWriter = commitTable.getWriter()) { 40 | 41 | // Test client 42 | try { 43 | commitTableClient.readLowWatermark().get(); 44 | } catch (UnsupportedOperationException e) { 45 | // expected 46 | } 47 | 48 | try { 49 | commitTableClient.getCommitTimestamp(TEST_ST).get(); 50 | } catch (UnsupportedOperationException e) { 51 | // expected 52 | } 53 | 54 | try { 55 | commitTableClient.tryInvalidateTransaction(TEST_ST).get(); 56 | } catch (UnsupportedOperationException e) { 57 | // expected 58 | } 59 | 60 | assertNull(commitTableClient.completeTransaction(TEST_ST).get()); 61 | 62 | // Test writer 63 | commitTableWriter.updateLowWatermark(TEST_LWM); 64 | commitTableWriter.addCommittedTransaction(TEST_ST, TEST_CT); 65 | commitTableWriter.clearWriteBuffer(); 66 | commitTableWriter.flush(); 67 | } 68 | } 69 | 70 | } 71 | -------------------------------------------------------------------------------- /common/src/main/java/org/apache/omid/NetworkUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid; 19 | 20 | import org.slf4j.Logger; 21 | import org.slf4j.LoggerFactory; 22 | 23 | import java.net.NetworkInterface; 24 | import java.net.SocketException; 25 | import java.util.Enumeration; 26 | 27 | public class NetworkUtils { 28 | 29 | private static final Logger LOG = LoggerFactory.getLogger(NetworkUtils.class); 30 | 31 | private static final String LINUX_TSO_NET_IFACE_PREFIX = "eth"; 32 | private static final String MAC_TSO_NET_IFACE_PREFIX = "en"; 33 | 34 | public static String getDefaultNetworkInterface() { 35 | 36 | try { 37 | Enumeration networkInterfaces = NetworkInterface.getNetworkInterfaces(); 38 | while (networkInterfaces.hasMoreElements()) { 39 | String name = networkInterfaces.nextElement().getDisplayName(); 40 | LOG.info("Iterating over network interfaces, found '{}'", name); 41 | if (name.startsWith(MAC_TSO_NET_IFACE_PREFIX) || name.startsWith(LINUX_TSO_NET_IFACE_PREFIX)) { 42 | return name; 43 | } 44 | } 45 | } catch (SocketException ignored) { 46 | throw new RuntimeException("Failed to find any network interfaces", ignored); 47 | } 48 | throw new IllegalArgumentException(String.format("No network '%s*'/'%s*' interfaces found", 49 | MAC_TSO_NET_IFACE_PREFIX, LINUX_TSO_NET_IFACE_PREFIX)); 50 | 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /common/src/main/java/org/apache/omid/zk/ZKUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.zk; 19 | 20 | import org.apache.curator.RetryPolicy; 21 | import org.apache.curator.framework.CuratorFramework; 22 | import org.apache.curator.framework.CuratorFrameworkFactory; 23 | import org.apache.curator.retry.ExponentialBackoffRetry; 24 | import org.slf4j.Logger; 25 | import org.slf4j.LoggerFactory; 26 | 27 | import java.io.IOException; 28 | import java.util.concurrent.TimeUnit; 29 | 30 | public class ZKUtils { 31 | 32 | private static final Logger LOG = LoggerFactory.getLogger(ZKUtils.class); 33 | 34 | public static CuratorFramework initZKClient(String zkCluster, String namespace, int zkConnectionTimeoutInSec) 35 | throws IOException { 36 | 37 | LOG.info("Creating Zookeeper Client connecting to {}", zkCluster); 38 | 39 | RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3); 40 | CuratorFramework zkClient = CuratorFrameworkFactory.builder() 41 | .namespace(namespace) 42 | .connectString(zkCluster) 43 | .retryPolicy(retryPolicy) 44 | .build(); 45 | 46 | zkClient.start(); 47 | 48 | try { 49 | if (zkClient.blockUntilConnected(zkConnectionTimeoutInSec, TimeUnit.SECONDS)) { 50 | LOG.info("Connected to ZK cluster '{}', client in state: [{}]", zkCluster, zkClient.getState()); 51 | } else { 52 | String errorMsg = String.format("Can't contact ZK cluster '%s' after %d seconds", 53 | zkCluster, zkConnectionTimeoutInSec); 54 | throw new IOException(errorMsg); 55 | } 56 | } catch (InterruptedException ex) { 57 | throw new IOException(String.format("Interrupted whilst connecting to ZK cluster '%s'", zkCluster)); 58 | } 59 | 60 | return zkClient; 61 | 62 | } 63 | 64 | } 65 | -------------------------------------------------------------------------------- /common/src/main/proto/TSOProto.proto: -------------------------------------------------------------------------------- 1 | // 2 | // Licensed to the Apache Software Foundation (ASF) under one 3 | // or more contributor license agreements. See the NOTICE file 4 | // distributed with this work for additional information 5 | // regarding copyright ownership. The ASF licenses this file 6 | // to you under the Apache License, Version 2.0 (the 7 | // "License"); you may not use this file except in compliance 8 | // with the License. You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | option java_package = "org.apache.omid.proto"; 20 | 21 | option optimize_for = SPEED; 22 | 23 | message Request { 24 | optional TimestampRequest timestampRequest = 1; 25 | optional CommitRequest commitRequest = 2; 26 | optional HandshakeRequest handshakeRequest = 3; 27 | } 28 | 29 | message TimestampRequest { 30 | } 31 | 32 | message CommitRequest { 33 | optional int64 startTimestamp = 1; 34 | optional bool isRetry = 2 [default = false]; 35 | repeated int64 cellId = 3; 36 | } 37 | 38 | message Response { 39 | optional TimestampResponse timestampResponse = 1; 40 | optional CommitResponse commitResponse = 2; 41 | optional HandshakeResponse handshakeResponse = 3; 42 | } 43 | 44 | message TimestampResponse { 45 | optional int64 startTimestamp = 1; 46 | } 47 | 48 | message CommitResponse { 49 | optional bool aborted = 1; 50 | optional int64 startTimestamp = 2; 51 | optional int64 commitTimestamp = 3; 52 | } 53 | 54 | message Capabilities { 55 | // place here the capabilities a client has to have 56 | // to pass the handshake 57 | } 58 | 59 | message HandshakeRequest { 60 | optional Capabilities clientCapabilities = 1; 61 | } 62 | 63 | message HandshakeResponse { 64 | optional bool clientCompatible = 1; 65 | optional Capabilities serverCapabilities = 2; 66 | } 67 | -------------------------------------------------------------------------------- /common/src/test/java/org/apache/omid/YAMLUtilsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid; 19 | 20 | import org.testng.Assert; 21 | import org.testng.annotations.Test; 22 | 23 | import java.util.HashMap; 24 | import java.util.Map; 25 | 26 | public class YAMLUtilsTest { 27 | 28 | @Test(timeOut = 10_000) 29 | public void testLoadDefaultSettings_setToBean() throws Exception { 30 | Map map = new HashMap(); 31 | new YAMLUtils().loadSettings("test.yml", "default-test.yml", map); 32 | Assert.assertNotNull(map); 33 | Assert.assertEquals(map.get("prop1"), 11); 34 | Assert.assertEquals(map.get("prop2"), "22"); 35 | Assert.assertEquals(map.get("prop3"), 3); 36 | } 37 | 38 | @Test(timeOut = 10_000) 39 | public void testLoadDefaultSettings_setToBean2() throws Exception { 40 | Map map = new HashMap(); 41 | new YAMLUtils().loadSettings("test.yml", map); 42 | Assert.assertNotNull(map); 43 | Assert.assertEquals(map.get("prop1"), 11); 44 | Assert.assertEquals(map.get("prop2"), "22"); 45 | Assert.assertEquals(map.size(), 2); 46 | } 47 | 48 | } -------------------------------------------------------------------------------- /common/src/test/resources/default-test.yml: -------------------------------------------------------------------------------- 1 | prop1: 1 2 | prop2: "2" 3 | prop3: 3 -------------------------------------------------------------------------------- /common/src/test/resources/test.yml: -------------------------------------------------------------------------------- 1 | prop1: 11 2 | prop2: "22" -------------------------------------------------------------------------------- /doc/images/ModuleDependencies.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YahooArchive/omid/7d3ad63ef48ab1738e05889bbf30db190f1c0e56/doc/images/ModuleDependencies.png -------------------------------------------------------------------------------- /doc/images/omid-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YahooArchive/omid/7d3ad63ef48ab1738e05889bbf30db190f1c0e56/doc/images/omid-logo.png -------------------------------------------------------------------------------- /doc/site/markdown/basic-algorithm.md: -------------------------------------------------------------------------------- 1 | # Basic Transaction Algorithm 2 | 3 | A simplified version of the algorithm for performing transactional operations in a non-faulty scenario is depicted 4 | in the following figure: 5 | 6 | ![Basic algorithm for managing transactions](images/basic-alg.png) 7 | 8 | Initially, a user application (not shown in the figure for the sake of simplicity) starts a new transaction using 9 | the Transactional Client API, which in turn acquires a start timestamp from the TSO. 10 | 11 | Then, the user application logic, also through the Transactional Client API, performs a series of read and write 12 | operations in the snapshot provided by the transactional context recently acquired. 13 | 14 | In the case of reads, if the cells read have shadow cells and their value is in the snapshot for the transaction, 15 | their value is taken directly. If the commit timestamp is missed in the shadow cell, the Transactional Client will 16 | try to find it in the commit table. If the commit timestamp is found in there and the cell value is in the snapshot, 17 | the cell value is taken. However, if the commit timestamp is missed, the shadow cell is re-checked. If it still does 18 | not exist, the cell is ignored and another version that matches the snapshot will be retrieved from the datastore. 19 | 20 | When the application commits the transaction, the Transactional Client contacts the TSO in order to check the possible 21 | conflicts of the transaction. If the writeset does not conflict with other concurrent transactions, the transaction is 22 | committed, the TSO updates the commit table and replies back to the Transactional Client, which in turn, returns the 23 | control to the user application. 24 | 25 | Finally, the Transactional Client, on receiving the commit acknowledgement, updates the shadow cells with the required 26 | data. After this, it can also remove safely the entry added by the TSO in the commit table. 27 | 28 | In case the TSO detects conflicts, the transaction is aborted (not shown in the figure), the Transactional Client will 29 | clean up the data written to the datastore, and will inform the user application throwing a rollback exception. -------------------------------------------------------------------------------- /doc/site/markdown/client-failure-management.md: -------------------------------------------------------------------------------- 1 | # Management of Client Failures 2 | 3 | Upon user application crashes, Transactional Clients may leave some orphaned data in the datastore, even though 4 | the data in the datastore is kept in a consistent state. When another Transactional Client comes across this data, 5 | it will check whether the associated transaction was committed or not. If the data belongs to a transaction which 6 | has not been committed, and the transaction id is lower than the low watermark, then the data is deleted, since it 7 | belongs to a transaction that can never be committed. 8 | 9 | If data belongs to a transaction that was already committed, this means that the Transactional Client crashed between 10 | sending the commit request and completing the transaction in the commit table. This means that shadow cells have not 11 | been deleted. The Transactional Client reading will write the shadow cell for the orphan data. However, it is not able 12 | to complete the transaction in the commit table as it can not guarantee that shadow cells have been written for all cells 13 | in the transaction, since only the crashed client knew which cells it had written to. Consequently, this means that the 14 | commit table can grow indefinitely. Initially, this flaw should not cause many problems, since: 15 | 16 | * the amount of data stored is small (16 bytes) 17 | * the number of clients expected to fail between commit and completion is low 18 | * the data is stored on persistent storage. It is not bounded by memory. 19 | 20 | For example, if 1% of all Transactional Clients are expected to crash at this exact point in time, while the system is 21 | writing, on average, 100,000 transactions per second, the amount of data required to store these commit table entries 22 | for a whole year would be around 500 GB. A single disk could hold this. 23 | 24 | In any case, it would be desirable a mechanism to sanitize the Commit Table. If we can guarantee that all orphan data 25 | has been cleaned-up up to a certain low watermark, then all commit table entries whose start timestamp is lower than 26 | this low watermark can be deleted. 27 | 28 | In the case of HBase, there is already have a compactor which proactively cleans up transactions older than the low 29 | watermark. This could easily be extended to store the low watermark for each region it compacts, so then another process 30 | could clean up the commit table based on the minimum low watermark that was stored. -------------------------------------------------------------------------------- /doc/site/markdown/mailing-lists.md: -------------------------------------------------------------------------------- 1 | # Mailing Lists @ Apache Incubator 2 | 3 | | Developers | Commits | 4 | | -------------------------------- | --------------------------------------| 5 | | dev@omid.incubator.apache.org | commits@omid.incubator.apache.org | 6 | 7 | ## Subscribe 8 | 9 | In order to subscribe to a list, please add the '-subscribe' suffix to the identifier in each list and send an email. 10 | For example use dev-subscribe@omid.incubator.apache.org to subscribe to the list of developers. 11 | 12 | ## Unsubscribe 13 | Use the '-unsubscribe' suffix to unsubscribe from a list. -------------------------------------------------------------------------------- /doc/site/resources/css/site.css: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | /* Basics */ 19 | 20 | a { 21 | color: #B9140A; 22 | } 23 | 24 | body { 25 | font-size: 16px; 26 | font-family: 'Ubuntu', sans-serif; 27 | } 28 | 29 | /* Navigation Bar */ 30 | 31 | .navbar .container { 32 | color: white; 33 | background-color: #B9140A; 34 | background-image: none; 35 | } 36 | 37 | .navbar .nav>li>a { 38 | color: white; 39 | text-shadow: 0px 0px 10px black; 40 | } 41 | 42 | .navbar .nav li.dropdown.open>.dropdown-toggle { 43 | color: #08C; 44 | text-shadow: none; 45 | } 46 | 47 | .navbar .nav>li>a:focus, .navbar .nav>li>a:hover, .navbar .nav>li>a:active { 48 | color: white; 49 | text-shadow: 0px 0px 20px white; 50 | } 51 | 52 | 53 | /* Tables */ 54 | 55 | table td, table th { 56 | border: 1px solid #333; 57 | padding: 0 .5em; 58 | } 59 | 60 | table th { 61 | background: #CCC; 62 | padding: 0 .5em; 63 | } 64 | 65 | table { 66 | margin-top: 20px; 67 | margin-bottom: 20px; 68 | } 69 | 70 | 71 | /* Footer */ 72 | 73 | footer { 74 | color: white; 75 | text-shadow: 0px 0px 10px black; 76 | background-color: #B9140A; 77 | } 78 | 79 | footer .container { 80 | background-color: #B9140A; 81 | background-image: none; 82 | } 83 | 84 | footer .container a { 85 | color: white; 86 | text-shadow: 0px 0px 10px black; 87 | text-decoration: underline; 88 | } 89 | 90 | footer .pull-right { 91 | color: white; 92 | text-shadow: 0px 0px 10px black; 93 | } 94 | 95 | /* Others */ 96 | 97 | .row { 98 | margin-top: 20px; 99 | } 100 | 101 | .breadcrumb { 102 | padding: 20px 15px; 103 | background-color: #B9140A; 104 | } 105 | 106 | 107 | #search-form { 108 | margin-left: 9px; 109 | margin-right: 40px; 110 | } 111 | 112 | #publishDate { 113 | color: white; 114 | text-shadow: 0px 0px 10px black; 115 | } 116 | 117 | #projectVersion { 118 | color: white; 119 | text-shadow: 0px 0px 10px black; 120 | } -------------------------------------------------------------------------------- /doc/site/resources/images/about-omid.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YahooArchive/omid/7d3ad63ef48ab1738e05889bbf30db190f1c0e56/doc/site/resources/images/about-omid.png -------------------------------------------------------------------------------- /doc/site/resources/images/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YahooArchive/omid/7d3ad63ef48ab1738e05889bbf30db190f1c0e56/doc/site/resources/images/architecture.png -------------------------------------------------------------------------------- /doc/site/resources/images/basic-alg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YahooArchive/omid/7d3ad63ef48ab1738e05889bbf30db190f1c0e56/doc/site/resources/images/basic-alg.png -------------------------------------------------------------------------------- /doc/site/resources/images/contact.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YahooArchive/omid/7d3ad63ef48ab1738e05889bbf30db190f1c0e56/doc/site/resources/images/contact.png -------------------------------------------------------------------------------- /doc/site/resources/images/getting-started.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YahooArchive/omid/7d3ad63ef48ab1738e05889bbf30db190f1c0e56/doc/site/resources/images/getting-started.png -------------------------------------------------------------------------------- /doc/site/resources/images/omid-logo-transparent.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YahooArchive/omid/7d3ad63ef48ab1738e05889bbf30db190f1c0e56/doc/site/resources/images/omid-logo-transparent.png -------------------------------------------------------------------------------- /doc/site/resources/images/omid-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YahooArchive/omid/7d3ad63ef48ab1738e05889bbf30db190f1c0e56/doc/site/resources/images/omid-logo.png -------------------------------------------------------------------------------- /doc/site/resources/images/snapshot-isolation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YahooArchive/omid/7d3ad63ef48ab1738e05889bbf30db190f1c0e56/doc/site/resources/images/snapshot-isolation.png -------------------------------------------------------------------------------- /doc/site/resources/images/technical-documentation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YahooArchive/omid/7d3ad63ef48ab1738e05889bbf30db190f1c0e56/doc/site/resources/images/technical-documentation.png -------------------------------------------------------------------------------- /examples/maven/assembly/assembly.xml: -------------------------------------------------------------------------------- 1 | 4 | 5 | bin 6 | 7 | 8 | tar.gz 9 | 10 | 11 | true 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | /lib 20 | false 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | ${basedir}/*.sh 30 | 31 | 32 | 33 | 34 | 35 | 36 | target/${project.artifactId}-${project.version}.jar 37 | / 38 | omid-examples.jar 39 | 40 | 41 | ${basedir}/src/main/resources/log4j.properties 42 | /conf 43 | log4j.properties 44 | 45 | 46 | ${basedir}/src/main/resources/hbase-omid-client-config.yml 47 | /conf 48 | hbase-omid-client-config.yml 49 | 50 | 51 | 52 | 53 | 54 | /lib 55 | false 56 | runtime 57 | false 58 | 59 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /examples/src/main/java/org/apache/omid/examples/RowIdGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.examples; 19 | 20 | interface RowIdGenerator { 21 | 22 | byte[] getRowId(); 23 | } 24 | -------------------------------------------------------------------------------- /examples/src/main/resources/hbase-omid-client-config.yml: -------------------------------------------------------------------------------- 1 | # Secure HBase credentials, core-site.xml and hbase-site.xml are expected to be in the classpath 2 | # It's used for secure HBase only and ignored otherwise 3 | # principal: omid_hbase_client 4 | # keytab: /path/to/hbase/client/keytab 5 | 6 | omidClientConfiguration: !!org.apache.omid.tso.client.OmidClientConfiguration 7 | connectionString: "localhost:54758" 8 | 9 | metrics: !!org.apache.omid.metrics.CodahaleMetricsProvider [ 10 | !!org.apache.omid.metrics.CodahaleMetricsConfig { 11 | outputFreqInSecs: 1, 12 | reporters: !!set { 13 | !!org.apache.omid.metrics.CodahaleMetricsConfig$Reporter CSV, 14 | !!org.apache.omid.metrics.CodahaleMetricsConfig$Reporter CONSOLE 15 | }, 16 | csvDir: "csvMetrics", 17 | prefix: "somePrefix", 18 | } 19 | ] 20 | 21 | -------------------------------------------------------------------------------- /examples/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO,console 2 | log4j.logger.org.apache.omid.examples=INFO 3 | 4 | # Logging Threshold 5 | log4j.threshold=ALL 6 | 7 | log4j.appender.console=org.apache.log4j.ConsoleAppender 8 | log4j.appender.console.target=System.err 9 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 10 | log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} [%t] %p %c{2}: %m%n 11 | -------------------------------------------------------------------------------- /hbase-client/src/main/java/org/apache/omid/transaction/ColumnWrapper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.transaction; 19 | 20 | import java.util.Arrays; 21 | 22 | public class ColumnWrapper { 23 | private byte[] family; 24 | private byte[] qualifier; 25 | 26 | public ColumnWrapper(byte[] family, byte[] qualifier) { 27 | this.family = family; 28 | this.qualifier = qualifier; 29 | } 30 | 31 | public byte[] getFamily() { 32 | return family; 33 | } 34 | 35 | public byte[] getQualifier() { 36 | return qualifier; 37 | } 38 | 39 | @Override 40 | public int hashCode() { 41 | final int prime = 31; 42 | int result = 1; 43 | result = prime * result + Arrays.hashCode(family); 44 | result = prime * result + Arrays.hashCode(qualifier); 45 | return result; 46 | } 47 | 48 | @Override 49 | public boolean equals(Object obj) { 50 | if (this == obj) 51 | return true; 52 | if (obj == null) 53 | return false; 54 | if (getClass() != obj.getClass()) 55 | return false; 56 | ColumnWrapper other = (ColumnWrapper) obj; 57 | if (!Arrays.equals(family, other.family)) 58 | return false; 59 | return Arrays.equals(qualifier, other.qualifier); 60 | } 61 | 62 | } -------------------------------------------------------------------------------- /hbase-client/src/main/java/org/apache/omid/transaction/HBaseAsyncPostCommitter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.transaction; 19 | 20 | import com.google.common.util.concurrent.ListenableFuture; 21 | import com.google.common.util.concurrent.ListeningExecutorService; 22 | import org.apache.omid.tso.client.CellId; 23 | import org.slf4j.Logger; 24 | import org.slf4j.LoggerFactory; 25 | 26 | import java.util.concurrent.Callable; 27 | 28 | public class HBaseAsyncPostCommitter implements PostCommitActions { 29 | 30 | private static final Logger LOG = LoggerFactory.getLogger(HBaseAsyncPostCommitter.class); 31 | 32 | private PostCommitActions syncPostCommitter; 33 | 34 | private ListeningExecutorService postCommitExecutor; 35 | 36 | public HBaseAsyncPostCommitter(PostCommitActions postCommitter, ListeningExecutorService postCommitExecutor) { 37 | this.syncPostCommitter = postCommitter; 38 | this.postCommitExecutor = postCommitExecutor; 39 | } 40 | 41 | @Override 42 | public ListenableFuture updateShadowCells(final AbstractTransaction transaction) { 43 | 44 | return postCommitExecutor.submit(new Callable() { 45 | @Override 46 | public Void call() throws Exception { 47 | syncPostCommitter.updateShadowCells(transaction); 48 | return null; 49 | } 50 | 51 | }); 52 | 53 | } 54 | 55 | @Override 56 | public ListenableFuture removeCommitTableEntry(final AbstractTransaction transaction) { 57 | 58 | return postCommitExecutor.submit(new Callable() { 59 | @Override 60 | public Void call() throws Exception { 61 | syncPostCommitter.removeCommitTableEntry(transaction); 62 | return null; 63 | } 64 | }); 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /hbase-client/src/main/java/org/apache/omid/transaction/HBaseCellId.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.transaction; 19 | 20 | import com.google.common.hash.Hashing; 21 | import org.apache.omid.tso.client.CellId; 22 | import org.apache.hadoop.hbase.client.HTableInterface; 23 | 24 | import static com.google.common.base.Charsets.UTF_8; 25 | 26 | public class HBaseCellId implements CellId { 27 | 28 | private final HTableInterface table; 29 | private final byte[] row; 30 | private final byte[] family; 31 | private final byte[] qualifier; 32 | private long timestamp; 33 | 34 | public HBaseCellId(HTableInterface table, byte[] row, byte[] family, byte[] qualifier, long timestamp) { 35 | this.timestamp = timestamp; 36 | this.table = table; 37 | this.row = row; 38 | this.family = family; 39 | this.qualifier = qualifier; 40 | } 41 | 42 | public HTableInterface getTable() { 43 | return table; 44 | } 45 | 46 | public byte[] getRow() { 47 | return row; 48 | } 49 | 50 | public byte[] getFamily() { 51 | return family; 52 | } 53 | 54 | public byte[] getQualifier() { 55 | return qualifier; 56 | } 57 | 58 | public long getTimestamp() { 59 | return timestamp; 60 | } 61 | 62 | public String toString() { 63 | return new String(table.getTableName(), UTF_8) 64 | + ":" + new String(row, UTF_8) 65 | + ":" + new String(family, UTF_8) 66 | + ":" + new String(qualifier, UTF_8) 67 | + ":" + timestamp; 68 | } 69 | 70 | @Override 71 | public long getCellId() { 72 | return Hashing.murmur3_128().newHasher() 73 | .putBytes(table.getTableName()) 74 | .putBytes(row) 75 | .putBytes(family) 76 | .putBytes(qualifier) 77 | .hash().asLong(); 78 | } 79 | 80 | } 81 | -------------------------------------------------------------------------------- /hbase-client/src/main/java/org/apache/omid/transaction/HBaseTransactionClient.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.transaction; 19 | 20 | public interface HBaseTransactionClient { 21 | boolean isCommitted(HBaseCellId hBaseCellId) throws TransactionException; 22 | 23 | long getLowWatermark() throws TransactionException; 24 | } 25 | -------------------------------------------------------------------------------- /hbase-client/src/main/java/org/apache/omid/transaction/TTableCellGetterAdapter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.transaction; 19 | 20 | import org.apache.omid.transaction.CellUtils.CellGetter; 21 | import org.apache.hadoop.hbase.client.Get; 22 | import org.apache.hadoop.hbase.client.Result; 23 | 24 | import java.io.IOException; 25 | 26 | public class TTableCellGetterAdapter implements CellGetter { 27 | 28 | private final TTable txTable; 29 | 30 | public TTableCellGetterAdapter(TTable txTable) { 31 | this.txTable = txTable; 32 | } 33 | 34 | public Result get(Get get) throws IOException { 35 | return txTable.getHTable().get(get); 36 | } 37 | 38 | } -------------------------------------------------------------------------------- /hbase-client/src/main/resources/default-hbase-omid-client-config.yml: -------------------------------------------------------------------------------- 1 | #HBase related 2 | commitTableName: OMID_COMMIT_TABLE 3 | 4 | #TSO/HA connection 5 | omidClientConfiguration: !!org.apache.omid.tso.client.OmidClientConfiguration [ ] 6 | 7 | #Instrumentation 8 | metrics: !!org.apache.omid.metrics.NullMetricsProvider [ ] -------------------------------------------------------------------------------- /hbase-client/src/test/java/org/apache/omid/transaction/TestAutoFlush.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.transaction; 19 | 20 | import org.apache.hadoop.hbase.client.Get; 21 | import org.apache.hadoop.hbase.client.Put; 22 | import org.apache.hadoop.hbase.client.Result; 23 | import org.apache.hadoop.hbase.util.Bytes; 24 | import org.testng.ITestContext; 25 | import org.testng.annotations.Test; 26 | 27 | import static org.testng.Assert.assertEquals; 28 | 29 | @Test(groups = "sharedHBase") 30 | public class TestAutoFlush extends OmidTestBase { 31 | 32 | @Test(timeOut = 10_000) 33 | public void testReadWithSeveralUncommitted(ITestContext context) throws Exception { 34 | 35 | byte[] family = Bytes.toBytes(TEST_FAMILY); 36 | byte[] row = Bytes.toBytes("row"); 37 | byte[] col = Bytes.toBytes("col1"); 38 | byte[] data = Bytes.toBytes("data"); 39 | TransactionManager tm = newTransactionManager(context); 40 | TTable table = new TTable(hbaseConf, TEST_TABLE); 41 | 42 | // Turn off autoflush 43 | table.setAutoFlush(false); 44 | 45 | Transaction t = tm.begin(); 46 | Put put = new Put(row); 47 | put.add(family, col, data); 48 | table.put(t, put); 49 | 50 | // Data shouldn't be in DB yet 51 | Get get = new Get(row); 52 | Result result = table.getHTable().get(get); 53 | assertEquals(result.size(), 0, "Writes are already in DB"); 54 | 55 | tm.commit(t); 56 | 57 | // After commit, both the cell and shadow cell should be there. 58 | // That's why we check for two elements in the test assertion 59 | result = table.getHTable().get(get); 60 | assertEquals(result.size(), 2, "Writes were not flushed to DB"); 61 | } 62 | 63 | } 64 | -------------------------------------------------------------------------------- /hbase-client/src/test/java/org/apache/omid/transaction/TestHBaseOmidClientConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.transaction; 19 | 20 | import org.junit.Assert; 21 | import org.testng.annotations.Test; 22 | 23 | public class TestHBaseOmidClientConfiguration { 24 | 25 | @Test(timeOut = 10_000) 26 | public void testYamlReading() { 27 | HBaseOmidClientConfiguration configuration = new HBaseOmidClientConfiguration(); 28 | Assert.assertNotNull(configuration.getCommitTableName()); 29 | Assert.assertNotNull(configuration.getHBaseConfiguration()); 30 | Assert.assertNotNull(configuration.getMetrics()); 31 | Assert.assertNotNull(configuration.getOmidClientConfiguration()); 32 | } 33 | 34 | @Test(timeOut = 10_000) 35 | public void testYamlReadingFromFile() { 36 | HBaseOmidClientConfiguration configuration = new HBaseOmidClientConfiguration("/test-hbase-omid-client-config.yml"); 37 | Assert.assertNotNull(configuration.getCommitTableName()); 38 | Assert.assertNotNull(configuration.getHBaseConfiguration()); 39 | Assert.assertNotNull(configuration.getMetrics()); 40 | Assert.assertNotNull(configuration.getOmidClientConfiguration()); 41 | } 42 | 43 | } -------------------------------------------------------------------------------- /hbase-client/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO,console 2 | 3 | # Logging Threshold 4 | log4j.threshold=ALL 5 | 6 | # 7 | # Daily Rolling File Appender 8 | # 9 | log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender 10 | log4j.appender.DRFA.File=${omid.log.dir}/${omid.log.file} 11 | 12 | # Rollver at midnight 13 | log4j.appender.DRFA.DatePattern=.yyyy-MM-dd 14 | 15 | # 30-day backup 16 | #log4j.appender.DRFA.MaxBackupIndex=30 17 | log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout 18 | 19 | # Pattern format: Date LogLevel LoggerName LogMessage 20 | log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 21 | 22 | # Debugging Pattern format 23 | #log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n 24 | 25 | 26 | # 27 | # console 28 | # Add "console" to rootlogger above if you want to use this 29 | # 30 | log4j.appender.console=org.apache.log4j.ConsoleAppender 31 | log4j.appender.console.target=System.err 32 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 33 | log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss,SSS} [%t] %p %c{2}: %m%n 34 | 35 | # Custom Logging levels 36 | log4j.logger.org.apache.zookeeper=INFO 37 | log4j.logger.org.apache.hadoop=INFO 38 | log4j.logger.org.apache.hadoop.ipc=ERROR 39 | log4j.logger.org.apache.omid=INFO 40 | #log4j.logger.org.apache.omid.regionserver.TransactionalRegionServer=TRACE 41 | #log4j.logger.org.apache.omid.TestBasicTransaction=TRACE 42 | #log4j.logger.org.apache.omid.client.TSOClient=TRACE 43 | #log4j.logger.org.apache.omid.client.TransactionState=TRACE 44 | #log4j.logger.org.apache.omid.tso.ThroughputMonitor=TRACE 45 | #log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG 46 | 47 | # Make these two classes INFO-level. Make them DEBUG to see more zk debug. 48 | #log4j.logger.org.apache.hadoop.dfs=DEBUG 49 | # Set this class to log INFO only otherwise its OTT 50 | 51 | # Uncomment the below if you want to remove logging of client region caching' 52 | # and scan of .META. messages 53 | # log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=INFO 54 | # log4j.logger.org.apache.hadoop.hbase.client.MetaScanner=INFO 55 | -------------------------------------------------------------------------------- /hbase-client/src/test/resources/test-hbase-omid-client-config.yml: -------------------------------------------------------------------------------- 1 | #HBase related 2 | commitTableName: OMID_COMMIT_TABLE1 3 | 4 | #TSO/HA connection 5 | omidClientConfiguration: !!org.apache.omid.tso.client.OmidClientConfiguration 6 | #TSO/HA connection 7 | connectionString: "somehost:54758" 8 | connectionType: !!org.apache.omid.tso.client.OmidClientConfiguration$ConnType HA 9 | zkConnectionTimeoutInSecs: 11 10 | 11 | #TSO related 12 | requestMaxRetries: 6 13 | requestTimeoutInMs: 5001 14 | reconnectionDelayInSecs: 11 15 | retryDelayInMs: 1001 16 | executorThreads: 4 17 | 18 | #Instrumentation 19 | metrics: !!org.apache.omid.metrics.CodahaleMetricsProvider [ 20 | !!org.apache.omid.metrics.CodahaleMetricsConfig { 21 | outputFreqInSecs: 1, 22 | reporters: !!set { 23 | !!org.apache.omid.metrics.CodahaleMetricsConfig$Reporter CSV, 24 | !!org.apache.omid.metrics.CodahaleMetricsConfig$Reporter SLF4J, 25 | !!org.apache.omid.metrics.CodahaleMetricsConfig$Reporter GRAPHITE, 26 | !!org.apache.omid.metrics.CodahaleMetricsConfig$Reporter CONSOLE 27 | }, 28 | csvDir: "some/folder", 29 | prefix: "somePrefix", 30 | slf4jLogger: "org.apache", 31 | graphiteHostConfig: "somehost:1234" 32 | } 33 | ] 34 | 35 | -------------------------------------------------------------------------------- /hbase-commit-table/src/main/java/org/apache/omid/committable/hbase/HBaseCommitTableStorageModule.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.committable.hbase; 19 | 20 | import com.google.inject.AbstractModule; 21 | import org.apache.omid.committable.CommitTable; 22 | import org.apache.hadoop.conf.Configuration; 23 | 24 | import javax.inject.Singleton; 25 | 26 | public class HBaseCommitTableStorageModule extends AbstractModule { 27 | 28 | @Override 29 | public void configure() { 30 | 31 | requireBinding(Configuration.class); 32 | // HBase commit table creation 33 | bind(CommitTable.class).to(HBaseCommitTable.class).in(Singleton.class); 34 | 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /hbase-common/src/main/java/org/apache/omid/committable/hbase/KeyGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.committable.hbase; 19 | 20 | import java.io.IOException; 21 | 22 | /** 23 | * Implementations of this interface determine how keys are spread in HBase 24 | */ 25 | public interface KeyGenerator { 26 | byte[] startTimestampToKey(long startTimestamp) throws IOException; 27 | 28 | long keyToStartTimestamp(byte[] key) throws IOException; 29 | } 30 | -------------------------------------------------------------------------------- /hbase-common/src/main/java/org/apache/omid/tools/hbase/HBaseLogin.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.tools.hbase; 19 | 20 | import org.apache.hadoop.security.UserGroupInformation; 21 | import org.slf4j.Logger; 22 | import org.slf4j.LoggerFactory; 23 | 24 | import java.io.IOException; 25 | 26 | public final class HBaseLogin { 27 | 28 | private static final Logger LOG = LoggerFactory.getLogger(HBaseLogin.class); 29 | 30 | public static UserGroupInformation loginIfNeeded(SecureHBaseConfig config) throws IOException { 31 | if (UserGroupInformation.isSecurityEnabled()) { 32 | LOG.info("Security is enabled, logging in with principal={}, keytab={}", 33 | config.getPrincipal(), config.getKeytab()); 34 | UserGroupInformation.loginUserFromKeytab(config.getPrincipal(), config.getKeytab()); 35 | } 36 | return UserGroupInformation.getCurrentUser(); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /hbase-common/src/main/java/org/apache/omid/tools/hbase/SecureHBaseConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.tools.hbase; 19 | 20 | import com.google.inject.Inject; 21 | import com.google.inject.name.Named; 22 | 23 | public class SecureHBaseConfig { 24 | 25 | public static final String HBASE_CLIENT_PRINCIPAL_KEY = "hbase.client.principal"; 26 | public static final String HBASE_CLIENT_KEYTAB_KEY = "hbase.client.keytab"; 27 | 28 | private String principal = "not set"; 29 | private String keytab = "not set"; 30 | 31 | // ---------------------------------------------------------------------------------------------------------------- 32 | // WARNING: Do not remove getters/setters, needed by snake_yaml! 33 | // ---------------------------------------------------------------------------------------------------------------- 34 | 35 | public String getPrincipal() { 36 | return principal; 37 | } 38 | 39 | @Inject(optional = true) 40 | @Named(HBASE_CLIENT_PRINCIPAL_KEY) 41 | public void setPrincipal(String principal) { 42 | this.principal = principal; 43 | } 44 | 45 | public String getKeytab() { 46 | return keytab; 47 | } 48 | 49 | @Inject(optional = true) 50 | @Named(HBASE_CLIENT_KEYTAB_KEY) 51 | public void setKeytab(String keytab) { 52 | this.keytab = keytab; 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /hbase-common/src/main/java/org/apache/omid/transaction/CellInfo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.transaction; 19 | 20 | import com.google.common.base.Objects; 21 | import org.apache.hadoop.hbase.Cell; 22 | 23 | public class CellInfo { 24 | 25 | private final Cell cell; 26 | private final Cell shadowCell; 27 | private final long timestamp; 28 | 29 | public CellInfo(Cell cell, Cell shadowCell) { 30 | // TODO: Use Guava preconditions instead of the assertions 31 | assert (cell != null && shadowCell != null); 32 | assert (cell.getTimestamp() == shadowCell.getTimestamp()); 33 | this.cell = cell; 34 | this.shadowCell = shadowCell; 35 | this.timestamp = cell.getTimestamp(); 36 | } 37 | 38 | public Cell getCell() { 39 | return cell; 40 | } 41 | 42 | public Cell getShadowCell() { 43 | return shadowCell; 44 | } 45 | 46 | public long getTimestamp() { 47 | return timestamp; 48 | } 49 | 50 | @Override 51 | public String toString() { 52 | return Objects.toStringHelper(this) 53 | .add("ts", timestamp) 54 | .add("cell", cell) 55 | .add("shadow cell", shadowCell) 56 | .toString(); 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /hbase-common/src/test/java/org/apache/omid/committable/hbase/TestHBaseCommitTableKey.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.committable.hbase; 19 | 20 | import org.apache.omid.committable.hbase.KeyGeneratorImplementations.BadRandomKeyGenerator; 21 | import org.apache.omid.committable.hbase.KeyGeneratorImplementations.BucketKeyGenerator; 22 | import org.apache.omid.committable.hbase.KeyGeneratorImplementations.FullRandomKeyGenerator; 23 | import org.apache.omid.committable.hbase.KeyGeneratorImplementations.SeqKeyGenerator; 24 | import org.testng.annotations.Test; 25 | 26 | import java.io.IOException; 27 | 28 | import static java.lang.Long.MAX_VALUE; 29 | import static org.testng.Assert.assertEquals; 30 | 31 | public class TestHBaseCommitTableKey { 32 | 33 | @Test(timeOut = 10_000) 34 | public void testEncodeDecode() throws Exception { 35 | testKeyGen(new BucketKeyGenerator()); 36 | testKeyGen(new BadRandomKeyGenerator()); 37 | testKeyGen(new FullRandomKeyGenerator()); 38 | testKeyGen(new SeqKeyGenerator()); 39 | } 40 | 41 | @Test(enabled = false, timeOut = 10_000) 42 | private void testKeyGen(KeyGenerator keyGen) throws IOException { 43 | assertEquals(keyGen.keyToStartTimestamp(keyGen.startTimestampToKey(0)), 0, "Should match"); 44 | assertEquals(keyGen.keyToStartTimestamp(keyGen.startTimestampToKey(1)), 1, "Should match"); 45 | assertEquals(keyGen.keyToStartTimestamp(keyGen.startTimestampToKey(8)), 8, "Should match"); 46 | assertEquals(keyGen.keyToStartTimestamp(keyGen.startTimestampToKey(1024)), 1024, "Should match"); 47 | assertEquals(keyGen.keyToStartTimestamp(keyGen.startTimestampToKey(1234)), 1234, "Should match"); 48 | assertEquals(keyGen.keyToStartTimestamp(keyGen.startTimestampToKey(4321)), 4321, "Should match"); 49 | assertEquals(keyGen.keyToStartTimestamp(keyGen.startTimestampToKey(0xdeadbeefcafeL)), 0xdeadbeefcafeL, "Should match"); 50 | assertEquals(keyGen.keyToStartTimestamp(keyGen.startTimestampToKey(MAX_VALUE)), MAX_VALUE, "Should match"); 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /hbase-coprocessor/.gitignore: -------------------------------------------------------------------------------- 1 | /bin 2 | -------------------------------------------------------------------------------- /hbase-coprocessor/src/main/java/org/apache/omid/transaction/HRegionCellGetterAdapter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.transaction; 19 | 20 | import org.apache.omid.transaction.CellUtils.CellGetter; 21 | import org.apache.hadoop.hbase.client.Get; 22 | import org.apache.hadoop.hbase.client.Result; 23 | import org.apache.hadoop.hbase.regionserver.HRegion; 24 | 25 | import java.io.IOException; 26 | 27 | public class HRegionCellGetterAdapter implements CellGetter { 28 | 29 | private final HRegion hRegion; 30 | 31 | public HRegionCellGetterAdapter(HRegion hRegion) { 32 | this.hRegion = hRegion; 33 | } 34 | 35 | public Result get(Get get) throws IOException { 36 | return hRegion.get(get); 37 | } 38 | 39 | } -------------------------------------------------------------------------------- /hbase-coprocessor/src/test/resources/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | -------------------------------------------------------------------------------- /hbase-shims/hbase-0/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | 6 | org.apache.omid 7 | omid-shims-aggregator 8 | 0.8.2.11-SNAPSHOT 9 | 10 | 11 | omid-hbase0-shims 12 | Shims layer for HBase 0.x 13 | jar 14 | 15 | 16 | ${hbase0.version} 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /hbase-shims/hbase-0/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.hadoop.hbase.regionserver; 19 | 20 | import org.apache.hadoop.hbase.HRegionInfo; 21 | import org.apache.hadoop.hbase.client.Get; 22 | import org.apache.hadoop.hbase.client.Result; 23 | 24 | import java.io.IOException; 25 | 26 | public class Region { 27 | 28 | HRegion hRegion; 29 | 30 | public Region(HRegion hRegion) { 31 | 32 | this.hRegion = hRegion; 33 | 34 | } 35 | 36 | Result get(Get getOperation) throws IOException { 37 | 38 | return hRegion.get(getOperation); 39 | 40 | } 41 | 42 | HRegionInfo getRegionInfo() { 43 | 44 | return hRegion.getRegionInfo(); 45 | 46 | } 47 | } -------------------------------------------------------------------------------- /hbase-shims/hbase-0/src/main/java/org/apache/hadoop/hbase/regionserver/ScannerContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.hadoop.hbase.regionserver; 19 | 20 | public class ScannerContext { 21 | 22 | int getBatchLimit() { 23 | 24 | return -1; 25 | 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /hbase-shims/hbase-0/src/main/java/org/apache/omid/HBaseShims.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid; 19 | 20 | import org.apache.hadoop.hbase.HColumnDescriptor; 21 | import org.apache.hadoop.hbase.HTableDescriptor; 22 | import org.apache.hadoop.hbase.KeyValue; 23 | import org.apache.hadoop.hbase.TableName; 24 | import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; 25 | import org.apache.hadoop.hbase.regionserver.HRegion; 26 | import org.apache.hadoop.hbase.regionserver.HRegionServer; 27 | import org.apache.hadoop.hbase.regionserver.Region; 28 | 29 | import java.io.IOException; 30 | 31 | public class HBaseShims { 32 | 33 | static public void setKeyValueSequenceId(KeyValue kv, int sequenceId) { 34 | 35 | kv.setMvccVersion(sequenceId); 36 | 37 | } 38 | 39 | static public Region getRegionCoprocessorRegion(RegionCoprocessorEnvironment env) { 40 | 41 | return new Region(env.getRegion()); 42 | 43 | } 44 | 45 | static public void flushAllOnlineRegions(HRegionServer regionServer, TableName tableName) throws IOException { 46 | 47 | for (HRegion r : regionServer.getOnlineRegions(tableName)) { 48 | r.flushcache(); 49 | } 50 | 51 | } 52 | 53 | static public void addFamilyToHTableDescriptor(HTableDescriptor tableDesc, HColumnDescriptor columnDesc) { 54 | 55 | tableDesc.addFamily(columnDesc); 56 | 57 | } 58 | 59 | } -------------------------------------------------------------------------------- /hbase-shims/hbase-1/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | 6 | org.apache.omid 7 | omid-shims-aggregator 8 | 0.8.2.11-SNAPSHOT 9 | 10 | 11 | omid-hbase1-shims 12 | Shims layer for HBase 1.x 13 | jar 14 | 15 | 16 | ${hbase1.version} 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /hbase-shims/hbase-1/src/main/java/org/apache/omid/HBaseShims.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid; 19 | 20 | import org.apache.hadoop.hbase.HColumnDescriptor; 21 | import org.apache.hadoop.hbase.HTableDescriptor; 22 | import org.apache.hadoop.hbase.KeyValue; 23 | import org.apache.hadoop.hbase.TableName; 24 | import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; 25 | import org.apache.hadoop.hbase.regionserver.HRegionServer; 26 | import org.apache.hadoop.hbase.regionserver.Region; 27 | 28 | import java.io.IOException; 29 | 30 | public class HBaseShims { 31 | 32 | static public void setKeyValueSequenceId(KeyValue kv, int sequenceId) { 33 | 34 | kv.setSequenceId(sequenceId); 35 | 36 | } 37 | 38 | static public Region getRegionCoprocessorRegion(RegionCoprocessorEnvironment env) { 39 | 40 | return env.getRegion(); 41 | 42 | } 43 | 44 | static public void flushAllOnlineRegions(HRegionServer regionServer, TableName tableName) throws IOException { 45 | 46 | for (Region r : regionServer.getOnlineRegions(tableName)) { 47 | r.flush(true); 48 | } 49 | 50 | } 51 | 52 | static public void addFamilyToHTableDescriptor(HTableDescriptor tableDesc, HColumnDescriptor columnDesc) { 53 | 54 | tableDesc.addFamily(columnDesc); 55 | 56 | } 57 | 58 | } 59 | -------------------------------------------------------------------------------- /hbase-shims/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | 6 | org.apache.omid 7 | omid 8 | 0.8.2.11-SNAPSHOT 9 | 10 | 11 | omid-shims-aggregator 12 | pom 13 | Shims Aggregator for HBase 14 | 15 | 16 | hbase-0 17 | hbase-1 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | org.apache.hbase 26 | hbase-common 27 | 28 | 29 | org.apache.hbase 30 | hbase-client 31 | 32 | 33 | org.slf4j 34 | slf4j-log4j12 35 | 36 | 37 | 38 | 39 | org.apache.hbase 40 | hbase-server 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | org.apache.curator 49 | curator-framework 50 | ${curator.version} 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | org.slf4j 59 | slf4j-api 60 | ${slf4j.version} 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | maven-site-plugin 74 | 75 | true 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | -------------------------------------------------------------------------------- /metrics/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 4.0.0 5 | 6 | 7 | omid 8 | org.apache.omid 9 | 0.8.2.11-SNAPSHOT 10 | 11 | 12 | omid-metrics 13 | Metrics 14 | 15 | 16 | 17 | 18 | 19 | 20 | org.slf4j 21 | slf4j-api 22 | ${slf4j.version} 23 | 24 | 25 | org.slf4j 26 | slf4j-log4j12 27 | ${slf4j.version} 28 | 29 | 30 | log4j 31 | log4j 32 | ${log4j.version} 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | com.google.guava 41 | guava 42 | ${guava.version} 43 | 44 | 45 | com.google.inject 46 | guice 47 | ${guice.version} 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | maven-site-plugin 61 | 62 | true 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | -------------------------------------------------------------------------------- /metrics/src/main/java/org/apache/omid/metrics/AbstractMetricsConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.metrics; 19 | 20 | import com.google.inject.Inject; 21 | 22 | import javax.inject.Named; 23 | import javax.inject.Singleton; 24 | 25 | @Singleton 26 | public abstract class AbstractMetricsConfig { 27 | 28 | private static final int DEFAULT_OUTPUT_FREQ_IN_SECS = 60; 29 | 30 | private static final String OUTPUT_FREQ_IN_SECS_KEY = "metrics.output.frequency.secs"; 31 | 32 | private int outputFreqInSecs = DEFAULT_OUTPUT_FREQ_IN_SECS; 33 | 34 | public int getOutputFreqInSecs() { 35 | return outputFreqInSecs; 36 | } 37 | 38 | @Inject(optional = true) 39 | public void setOutputFreqInSecs(@Named(OUTPUT_FREQ_IN_SECS_KEY) int outputFreqInSecs) { 40 | this.outputFreqInSecs = outputFreqInSecs; 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /metrics/src/main/java/org/apache/omid/metrics/Counter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.metrics; 19 | 20 | public interface Counter extends Metric { 21 | 22 | /** 23 | * Increment the counter by one. 24 | */ 25 | void inc(); 26 | 27 | /** 28 | * Increment the counter by {@code n}. 29 | * 30 | * @param n the amount by which the counter will be increased 31 | */ 32 | void inc(long n); 33 | 34 | /** 35 | * Decrement the counter by one. 36 | */ 37 | void dec(); 38 | 39 | /** 40 | * Decrement the counter by {@code n}. 41 | * 42 | * @param n the amount by which the counter will be decreased 43 | */ 44 | void dec(long n); 45 | 46 | } -------------------------------------------------------------------------------- /metrics/src/main/java/org/apache/omid/metrics/Gauge.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.metrics; 19 | 20 | /** 21 | * A gauge returns the value of a metric measured at a specific point in time. 22 | * For example the current size of. The value of T must be some numeric type. 23 | */ 24 | public interface Gauge extends Metric { 25 | 26 | T getValue(); 27 | 28 | } 29 | -------------------------------------------------------------------------------- /metrics/src/main/java/org/apache/omid/metrics/Histogram.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.metrics; 19 | 20 | public interface Histogram extends Metric { 21 | 22 | /** 23 | * Adds a recorded value. 24 | * 25 | * @param value the length of the value 26 | */ 27 | void update(int value); 28 | 29 | /** 30 | * Adds a recorded value. 31 | * 32 | * @param value the length of the value 33 | */ 34 | void update(long value); 35 | 36 | } 37 | -------------------------------------------------------------------------------- /metrics/src/main/java/org/apache/omid/metrics/Meter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.metrics; 19 | 20 | public interface Meter extends Metric { 21 | 22 | /** 23 | * Mark the occurrence of an event. 24 | */ 25 | void mark(); 26 | 27 | /** 28 | * Mark the occurrence of a given number of events. 29 | * 30 | * @param n the number of events 31 | */ 32 | void mark(long n); 33 | 34 | } 35 | -------------------------------------------------------------------------------- /metrics/src/main/java/org/apache/omid/metrics/Metric.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.metrics; 19 | 20 | /** 21 | * Marker interface for distinguish metrics 22 | */ 23 | public interface Metric { 24 | 25 | } 26 | -------------------------------------------------------------------------------- /metrics/src/main/java/org/apache/omid/metrics/MetricsProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.metrics; 19 | 20 | /** 21 | * Provider to provide metrics logger for different scopes. 22 | */ 23 | public interface MetricsProvider { 24 | 25 | String CODAHALE_METRICS_CONFIG = "console:_:60:SECONDS"; 26 | 27 | enum Provider { 28 | CODAHALE, YMON 29 | } 30 | 31 | /** 32 | * Intialize the metrics provider. 33 | */ 34 | void startMetrics(); 35 | 36 | /** 37 | * Close the metrics provider. 38 | */ 39 | void stopMetrics(); 40 | 41 | } 42 | -------------------------------------------------------------------------------- /metrics/src/main/java/org/apache/omid/metrics/MetricsRegistry.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.metrics; 19 | 20 | public interface MetricsRegistry { 21 | 22 | /** 23 | * Registers the {@link Gauge} under the given name. 24 | * 25 | * @param name the name of the metric 26 | */ 27 | void gauge(String name, Gauge gauge); 28 | 29 | /** 30 | * Creates a new {@link Counter} and registers it under the given name. 31 | * 32 | * @param name the name of the metric 33 | * @return a new {@link Counter} 34 | */ 35 | Counter counter(String name); 36 | 37 | /** 38 | * Creates a new {@link Timer} and registers it under the given name. 39 | * 40 | * @param name the name of the metric 41 | * @return a new {@link Timer} 42 | */ 43 | Timer timer(String name); 44 | 45 | /** 46 | * Creates a new {@link Meter} and registers it under the given name. 47 | * 48 | * @param name the name of the metric 49 | * @return a new {@link Meter} 50 | */ 51 | Meter meter(String name); 52 | 53 | /** 54 | * Creates a new {@link Histogram} and registers it under the given name. 55 | * 56 | * @param name the name of the metric 57 | * @return a new {@link Histogram} 58 | */ 59 | Histogram histogram(String name); 60 | } 61 | -------------------------------------------------------------------------------- /metrics/src/main/java/org/apache/omid/metrics/MetricsUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.metrics; 19 | 20 | public class MetricsUtils { 21 | 22 | private static final char DEFAULT_SEPARATOR = '.'; 23 | 24 | public static String name(String name, String... otherNames) { 25 | return name(name, DEFAULT_SEPARATOR, otherNames); 26 | } 27 | 28 | public static String name(String name, char separator, String... otherNames) { 29 | final StringBuffer builder = new StringBuffer(name); 30 | if (otherNames != null) { 31 | for (String otherName : otherNames) { 32 | concat(builder, otherName, separator); 33 | } 34 | } 35 | return builder.toString(); 36 | } 37 | 38 | private static void concat(StringBuffer head, String tail, char separator) { 39 | if (tail != null && !tail.isEmpty()) { 40 | if (head.length() > 0) { 41 | head.append(separator); 42 | } 43 | head.append(tail); 44 | } 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /metrics/src/main/java/org/apache/omid/metrics/Timer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.metrics; 19 | 20 | public interface Timer extends Metric { 21 | 22 | void start(); 23 | 24 | void stop(); 25 | 26 | void update(long durationInNs); 27 | 28 | } 29 | -------------------------------------------------------------------------------- /misc/findbugs-exclude.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /misc/header.txt: -------------------------------------------------------------------------------- 1 | Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | -------------------------------------------------------------------------------- /packaging/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 15 | 16 | 17 | 4.0.0 18 | 19 | 20 | org.apache.omid 21 | omid 22 | 0.8.2.11-SNAPSHOT 23 | 24 | 25 | omid-packaging 26 | 0.8.2.1-SNAPSHOT 27 | pom 28 | Omid Packaging 29 | 30 | 31 | 32 | 33 | maven-assembly-plugin 34 | ${maven-assembly-plugin.version} 35 | 36 | gnu 37 | false 38 | apache-omid-incubating-${project.version} 39 | 40 | maven/assembly/src.xml 41 | 42 | 43 | 44 | 45 | package 46 | 47 | single 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | -------------------------------------------------------------------------------- /statemachine/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | 6 | org.apache.omid 7 | omid 8 | 0.8.2.11-SNAPSHOT 9 | 10 | 11 | State Machine 12 | omid-statemachine 13 | jar 14 | 15 | 16 | 17 | 18 | 19 | 20 | org.slf4j 21 | slf4j-api 22 | ${slf4j.version} 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | org.testng 31 | testng 32 | ${testng.version} 33 | test 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | maven-site-plugin 47 | 48 | true 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | -------------------------------------------------------------------------------- /timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/DefaultZKTimestampStorageModule.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.timestamp.storage; 19 | 20 | import com.google.inject.AbstractModule; 21 | 22 | /** 23 | * This class is instantiated by the yaml parser. 24 | * Snake_yaml needs a public POJO style class to work properly with all the setters and getters. 25 | */ 26 | public class DefaultZKTimestampStorageModule extends AbstractModule { 27 | 28 | private String zkCluster = "localhost:2181"; 29 | private String namespace = "omid"; 30 | 31 | @Override 32 | public void configure() { 33 | install(new ZKModule(zkCluster, namespace)); 34 | install(new ZKTimestampStorageModule()); 35 | } 36 | 37 | // ---------------------------------------------------------------------------------------------------------------- 38 | // WARNING: Do not remove getters/setters, needed by snake_yaml! 39 | // ---------------------------------------------------------------------------------------------------------------- 40 | 41 | public String getZkCluster() { 42 | return zkCluster; 43 | } 44 | 45 | public void setZkCluster(String zkCluster) { 46 | this.zkCluster = zkCluster; 47 | } 48 | 49 | public String getNamespace() { 50 | return namespace; 51 | } 52 | 53 | public void setNamespace(String namespace) { 54 | this.namespace = namespace; 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/HBaseTimestampStorageConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.timestamp.storage; 19 | 20 | import com.google.inject.Inject; 21 | 22 | import javax.inject.Named; 23 | 24 | public class HBaseTimestampStorageConfig { 25 | 26 | public static final String TIMESTAMP_STORAGE_TABLE_NAME_KEY = "omid.timestampstorage.tablename"; 27 | public static final String TIMESTAMP_STORAGE_CF_NAME_KEY = "omid.timestampstorage.cfname"; 28 | 29 | public static final String DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME = "OMID_TIMESTAMP_TABLE"; 30 | public static final String DEFAULT_TIMESTAMP_STORAGE_CF_NAME = "MAX_TIMESTAMP_CF"; 31 | 32 | // ---------------------------------------------------------------------------------------------------------------- 33 | // Configuration parameters 34 | // ---------------------------------------------------------------------------------------------------------------- 35 | 36 | private String tableName = DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME; 37 | private String familyName = DEFAULT_TIMESTAMP_STORAGE_CF_NAME; 38 | 39 | // ---------------------------------------------------------------------------------------------------------------- 40 | // Getters and setters 41 | // ---------------------------------------------------------------------------------------------------------------- 42 | 43 | public String getTableName() { 44 | return tableName; 45 | } 46 | 47 | @Inject(optional = true) 48 | public void setTableName(@Named(TIMESTAMP_STORAGE_TABLE_NAME_KEY) String tableName) { 49 | this.tableName = tableName; 50 | } 51 | 52 | public String getFamilyName() { 53 | return familyName; 54 | } 55 | 56 | @Inject(optional = true) 57 | public void setFamilyName(@Named(TIMESTAMP_STORAGE_CF_NAME_KEY) String familyName) { 58 | this.familyName = familyName; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/HBaseTimestampStorageModule.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.timestamp.storage; 19 | 20 | import com.google.inject.AbstractModule; 21 | import org.apache.hadoop.conf.Configuration; 22 | 23 | import javax.inject.Singleton; 24 | 25 | public class HBaseTimestampStorageModule extends AbstractModule { 26 | 27 | @Override 28 | public void configure() { 29 | 30 | requireBinding(Configuration.class); 31 | // Timestamp storage creation 32 | bind(TimestampStorage.class).to(HBaseTimestampStorage.class).in(Singleton.class); 33 | 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/TimestampStorage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.timestamp.storage; 19 | 20 | import java.io.IOException; 21 | 22 | public interface TimestampStorage { 23 | 24 | void updateMaxTimestamp(long previousMaxTimestamp, long newMaxTimestamp) throws IOException; 25 | 26 | long getMaxTimestamp() throws IOException; 27 | 28 | } 29 | -------------------------------------------------------------------------------- /timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/ZKModule.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.timestamp.storage; 19 | 20 | import com.google.inject.AbstractModule; 21 | import com.google.inject.Provides; 22 | import org.apache.curator.framework.CuratorFramework; 23 | import org.apache.omid.zk.ZKUtils; 24 | 25 | import javax.inject.Singleton; 26 | import java.io.IOException; 27 | 28 | //TODO:IK: move to common? 29 | public class ZKModule extends AbstractModule { 30 | 31 | private final String zkCluster; 32 | private final String namespace; 33 | 34 | public ZKModule(String zkCluster, String namespace) { 35 | this.zkCluster = zkCluster; 36 | this.namespace = namespace; 37 | } 38 | 39 | @Override 40 | public void configure() { 41 | } 42 | 43 | @Provides 44 | @Singleton 45 | CuratorFramework provideInitializedZookeeperClient() throws IOException { 46 | return ZKUtils.initZKClient(zkCluster, namespace, 10); 47 | } 48 | 49 | // ---------------------------------------------------------------------------------------------------------------- 50 | // NOTE: We need to implement equals() and hashCode() because the ZKModule is installed from several parent modules 51 | // ---------------------------------------------------------------------------------------------------------------- 52 | 53 | @Override 54 | public boolean equals(Object o) { 55 | 56 | if (this == o) return true; 57 | if (o == null || getClass() != o.getClass()) return false; 58 | 59 | ZKModule zkModule = (ZKModule) o; 60 | 61 | if (!zkCluster.equals(zkModule.zkCluster)) return false; 62 | return namespace.equals(zkModule.namespace); 63 | 64 | } 65 | 66 | @Override 67 | public int hashCode() { 68 | 69 | int result = zkCluster.hashCode(); 70 | result = 31 * result + namespace.hashCode(); 71 | return result; 72 | 73 | } 74 | 75 | } -------------------------------------------------------------------------------- /timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/ZKTimestampPaths.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.timestamp.storage; 19 | 20 | public class ZKTimestampPaths { 21 | 22 | public static final String TIMESTAMP_ZNODE = "/omid/timestamp"; 23 | 24 | // Avoid instantiation 25 | private ZKTimestampPaths() { 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/ZKTimestampStorageModule.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.timestamp.storage; 19 | 20 | import com.google.inject.AbstractModule; 21 | import org.apache.curator.framework.CuratorFramework; 22 | 23 | import javax.inject.Singleton; 24 | 25 | public class ZKTimestampStorageModule extends AbstractModule { 26 | 27 | 28 | @Override 29 | public void configure() { 30 | 31 | requireBinding(CuratorFramework.class); 32 | // Timestamp storage creation 33 | bind(TimestampStorage.class).to(ZKTimestampStorage.class).in(Singleton.class); 34 | 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /timestamp-storage/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO,console 2 | 3 | # Logging Threshold 4 | #log4j.threshold=INFO 5 | 6 | # 7 | # Daily Rolling File Appender 8 | # 9 | log4j.appender.DRFA.threshold=TRACE 10 | log4j.appender.DRFA=org.apache.log4j.FileAppender 11 | log4j.appender.DRFA.File=omid.log 12 | 13 | # Rollver at midnight 14 | #log4j.appender.DRFA.DatePattern=.yyyy-MM-dd 15 | log4j.appender.DRFA.append=false 16 | 17 | # 30-day backup 18 | #log4j.appender.DRFA.MaxBackupIndex=30 19 | log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout 20 | 21 | # Pattern format: Date LogLevel LoggerName LogMessage 22 | log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 23 | 24 | # Debugging Pattern format 25 | #log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n 26 | 27 | # 28 | # console 29 | # Add "console" to rootlogger above if you want to use this 30 | # 31 | log4j.appender.console=org.apache.log4j.ConsoleAppender 32 | log4j.appender.console.threshold=DEBUG 33 | log4j.appender.console.target=System.err 34 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 35 | log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} [%t] %p %c{2}: %m%n 36 | 37 | # Custom Logging levels 38 | log4j.logger.org.apache.zookeeper=ERROR 39 | log4j.logger.org.apache.bookkeeper=FATAL 40 | log4j.logger.org.apache.hadoop.hbase=ERROR 41 | log4j.logger.org.apache.hadoop.ipc=ERROR 42 | log4j.logger.org.apache.omid.util=DEBUG 43 | #log4j.logger.org.apache.omid.regionserver.TransactionalRegionServer=TRACE 44 | #log4j.logger.org.apache.omid.TestBasicTransaction=TRACE 45 | #log4j.logger.org.apache.omid.client.TSOClient=TRACE 46 | #log4j.logger.org.apache.omid.client.TransactionState=TRACE 47 | #log4j.logger.org.apache.omid.OmidTestBase=TRACE 48 | #log4j.logger.org.apache.omid.tso.ThroughputMonitor=INFO 49 | #log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG 50 | 51 | # Make these two classes INFO-level. Make them DEBUG to see more zk debug. 52 | #log4j.logger.org.apache.hadoop.dfs=DEBUG 53 | # Set this class to log INFO only otherwise its OTT 54 | 55 | # Uncomment the below if you want to remove logging of client region caching' 56 | # and scan of .META. messages 57 | # log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=INFO 58 | # log4j.logger.org.apache.hadoop.hbase.client.MetaScanner=INFO 59 | -------------------------------------------------------------------------------- /transaction-client/src/main/java/org/apache/omid/transaction/CommitTimestampLocator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.transaction; 19 | 20 | import com.google.common.base.Optional; 21 | 22 | import java.io.IOException; 23 | 24 | /** 25 | * An behavior that needs to be implemented by transaction managers 26 | * to try to locate the possible commit timestamp for a cell. It 27 | * allows to the transaction managers implementing this interface to 28 | * search the commit timestamp in a local cache of their own or in 29 | * their particular shadow cells implementation. 30 | */ 31 | public interface CommitTimestampLocator { 32 | 33 | Optional readCommitTimestampFromCache(long startTimestamp); 34 | 35 | Optional readCommitTimestampFromShadowCell(long startTimestamp) throws IOException; 36 | 37 | } 38 | -------------------------------------------------------------------------------- /transaction-client/src/main/java/org/apache/omid/transaction/PostCommitActions.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.transaction; 19 | 20 | import com.google.common.util.concurrent.ListenableFuture; 21 | import org.apache.omid.tso.client.CellId; 22 | 23 | public interface PostCommitActions { 24 | 25 | /** 26 | * Allows specific implementations to update the shadow cells. 27 | * @param transaction 28 | * the transaction to update shadow cells for 29 | * @return future signalling end of the computation 30 | */ 31 | ListenableFuture updateShadowCells(AbstractTransaction transaction); 32 | 33 | /** 34 | * Allows specific implementations to remove the transaction entry from the commit table. 35 | * @param transaction 36 | * the transaction to remove the commit table entry 37 | * @return future signalling end of the computation 38 | */ 39 | ListenableFuture removeCommitTableEntry(AbstractTransaction transaction); 40 | 41 | } 42 | -------------------------------------------------------------------------------- /transaction-client/src/main/java/org/apache/omid/transaction/RollbackException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.transaction; 19 | 20 | public class RollbackException extends Exception { 21 | 22 | private static final long serialVersionUID = -9163407697376986830L; 23 | 24 | public RollbackException(String message) { 25 | super(message); 26 | } 27 | 28 | public RollbackException(String message, Throwable cause) { 29 | super(message, cause); 30 | } 31 | 32 | public RollbackException() { 33 | super(); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /transaction-client/src/main/java/org/apache/omid/transaction/Transaction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.transaction; 19 | 20 | import com.google.common.base.Optional; 21 | 22 | /** 23 | * This interface defines the transaction state and behavior exposed to users. 24 | */ 25 | public interface Transaction { 26 | 27 | enum Status { 28 | RUNNING, COMMITTED, ROLLEDBACK, COMMITTED_RO 29 | } 30 | 31 | /** 32 | * Returns the transaction identifier 33 | * @return the transaction identifier 34 | */ 35 | long getTransactionId(); 36 | 37 | /** 38 | * Returns the epoch given by the TSOServer 39 | * @return the transaction's TSOServer epoch 40 | */ 41 | long getEpoch(); 42 | 43 | /** 44 | * Returns the current transaction {@link Status} 45 | * @return transaction status 46 | */ 47 | Status getStatus(); 48 | 49 | /** 50 | * Forces the transaction to rollback, even when there's an intention 51 | * to commit it. 52 | */ 53 | void setRollbackOnly(); 54 | 55 | /** 56 | * Returns whether the transaction was marked for rollback or not 57 | * @return whether the transaction is marked for rollback or not 58 | */ 59 | boolean isRollbackOnly(); 60 | 61 | 62 | /** 63 | * Set of methods to attach some metadata to a transaction object. One example 64 | * of such metadata are notifications 65 | * 66 | * 67 | * Expects they metadata stored under key "key" to be of the "Set" type, 68 | * append "value" to the existing set or creates a new one 69 | * @param key a key, like in hashtable 70 | * @param value a value to associate with the given key 71 | */ 72 | void appendMetadata(String key, Object value); 73 | 74 | 75 | void setMetadata(String key, Object value); 76 | 77 | Optional getMetadata(String key); 78 | } 79 | 80 | -------------------------------------------------------------------------------- /transaction-client/src/main/java/org/apache/omid/transaction/TransactionException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.transaction; 19 | 20 | import java.io.IOException; 21 | 22 | public class TransactionException extends IOException { 23 | 24 | private static final long serialVersionUID = 7273525983622126275L; 25 | 26 | public TransactionException(String reason) { 27 | super(reason); 28 | } 29 | 30 | public TransactionException(String reason, Throwable e) { 31 | super(reason, e); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /transaction-client/src/main/java/org/apache/omid/transaction/TransactionManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.transaction; 19 | 20 | import java.io.Closeable; 21 | 22 | /** 23 | * Provides the methods to manage transactions (create, commit...) 24 | */ 25 | public interface TransactionManager extends Closeable { 26 | 27 | /** 28 | * Starts a new transaction. 29 | * 30 | * Creates and returns a {@link Transaction} interface implementation that will be used in TTable's methods for 31 | * doing operations on the transactional context defined by the returned object. 32 | * 33 | * @return transaction representation of the created transaction 34 | * @throws TransactionException in case of any issues 35 | */ 36 | Transaction begin() throws TransactionException; 37 | 38 | /** 39 | * Commits a transaction. 40 | * 41 | * If the transaction was marked for rollback or has conflicts with another concurrent transaction it will be 42 | * rolledback automatically and a {@link RollbackException} will be thrown. 43 | * 44 | * @param tx transaction to be committed. 45 | * @throws RollbackException thrown when transaction has conflicts with another transaction or when was marked 46 | * for rollback. 47 | * @throws TransactionException in case of any issues 48 | */ 49 | void commit(Transaction tx) throws RollbackException, TransactionException; 50 | 51 | /** 52 | * Aborts a transaction. 53 | * 54 | * Automatically rollbacks the changes performed by the transaction. 55 | * 56 | * @param tx transaction to be rolled-back 57 | * @throws TransactionException in case of any issues 58 | */ 59 | void rollback(Transaction tx) throws TransactionException; 60 | 61 | } 62 | -------------------------------------------------------------------------------- /transaction-client/src/main/java/org/apache/omid/transaction/TransactionManagerException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.transaction; 19 | 20 | public class TransactionManagerException extends Exception { 21 | 22 | private static final long serialVersionUID = 6811947817962260774L; 23 | 24 | public TransactionManagerException(String reason) { 25 | super(reason); 26 | } 27 | 28 | public TransactionManagerException(String reason, Throwable e) { 29 | super(reason, e); 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /transaction-client/src/main/java/org/apache/omid/tso/client/AbortException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.tso.client; 19 | 20 | /** 21 | * Thrown when the TSO server has aborted a transaction 22 | */ 23 | public class AbortException extends Exception { 24 | 25 | private static final long serialVersionUID = 1861474360100681040L; 26 | 27 | } -------------------------------------------------------------------------------- /transaction-client/src/main/java/org/apache/omid/tso/client/CellId.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.tso.client; 19 | 20 | public interface CellId { 21 | 22 | long getCellId(); 23 | 24 | } -------------------------------------------------------------------------------- /transaction-client/src/main/java/org/apache/omid/tso/client/ClosingException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.tso.client; 19 | 20 | /** 21 | * Thrown when an error is produced when performing the actions required 22 | * to close the communication with the TSO server 23 | */ 24 | public class ClosingException extends Exception { 25 | 26 | private static final long serialVersionUID = -5681694952053689884L; 27 | 28 | } 29 | -------------------------------------------------------------------------------- /transaction-client/src/main/java/org/apache/omid/tso/client/ConnectionException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.tso.client; 19 | 20 | import java.io.IOException; 21 | 22 | /** 23 | * Thrown when there are problems with the comm channel with the TSO server 24 | * (e.g. when it is closed or disconnected) 25 | */ 26 | public class ConnectionException extends IOException { 27 | 28 | private static final long serialVersionUID = -8489539195700267443L; 29 | 30 | } 31 | -------------------------------------------------------------------------------- /transaction-client/src/main/java/org/apache/omid/tso/client/ForwardingTSOFuture.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.tso.client; 19 | 20 | import com.google.common.util.concurrent.ListenableFuture; 21 | 22 | import java.util.concurrent.ExecutionException; 23 | import java.util.concurrent.Executor; 24 | import java.util.concurrent.TimeUnit; 25 | import java.util.concurrent.TimeoutException; 26 | 27 | public class ForwardingTSOFuture implements TSOFuture { 28 | private final ListenableFuture future; 29 | 30 | public ForwardingTSOFuture(ListenableFuture future) { 31 | this.future = future; 32 | } 33 | 34 | @Override 35 | public boolean cancel(boolean mayInterruptIfRunning) { 36 | return future.cancel(mayInterruptIfRunning); 37 | } 38 | 39 | @Override 40 | public boolean isCancelled() { 41 | return future.isCancelled(); 42 | } 43 | 44 | @Override 45 | public boolean isDone() { 46 | return future.isDone(); 47 | } 48 | 49 | @Override 50 | public T get() throws InterruptedException, ExecutionException { 51 | return future.get(); 52 | } 53 | 54 | @Override 55 | public T get(long timeout, TimeUnit unit) 56 | throws InterruptedException, ExecutionException, TimeoutException { 57 | return future.get(timeout, unit); 58 | } 59 | 60 | @Override 61 | public void addListener(Runnable listener, Executor executor) { 62 | future.addListener(listener, executor); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /transaction-client/src/main/java/org/apache/omid/tso/client/HandshakeFailedException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.tso.client; 19 | 20 | /** 21 | * Thrown when some incompatibilities between the TSO client and server are 22 | * found 23 | */ 24 | public class HandshakeFailedException extends Exception { 25 | 26 | private static final long serialVersionUID = 8545505066920548834L; 27 | 28 | } 29 | -------------------------------------------------------------------------------- /transaction-client/src/main/java/org/apache/omid/tso/client/ServiceUnavailableException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.tso.client; 19 | 20 | /** 21 | * Thrown when the requests from TSO client to the TSO server have reached 22 | * a number of retries 23 | */ 24 | public class ServiceUnavailableException extends Exception { 25 | 26 | private static final long serialVersionUID = -1551974284011474385L; 27 | 28 | public ServiceUnavailableException(String message) { 29 | super(message); 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /transaction-client/src/main/java/org/apache/omid/tso/client/TSOFuture.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.tso.client; 19 | 20 | import java.util.concurrent.Executor; 21 | import java.util.concurrent.Future; 22 | 23 | public interface TSOFuture extends Future { 24 | void addListener(Runnable listener, Executor executor); 25 | } 26 | -------------------------------------------------------------------------------- /transaction-client/src/main/java/org/apache/omid/tso/client/TSOProtocol.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.tso.client; 19 | 20 | import java.util.Set; 21 | 22 | /** 23 | * Defines the protocol used on the client side to abstract communication to the TSO server 24 | */ 25 | public interface TSOProtocol { 26 | 27 | /** 28 | * Returns a new timestamp assigned by on the server-side 29 | * @return the newly assigned timestamp as a future. If an error was detected, the future will contain a 30 | * corresponding protocol exception 31 | * see org.apache.omid.tso.TimestampOracle 32 | * see org.apache.omid.tso.TSOServer 33 | */ 34 | TSOFuture getNewStartTimestamp(); 35 | 36 | /** 37 | * Returns the result of the conflict detection made on the server-side for the specified transaction 38 | * @param transactionId 39 | * the transaction to check for conflicts 40 | * @param writeSet 41 | * the writeSet of the transaction, which includes all the modified cells 42 | * @return the commit timestamp as a future if the transaction was committed. If the transaction was aborted due 43 | * to conflicts with a concurrent transaction, the future will include an AbortException. If an error was detected, 44 | * the future will contain a corresponding protocol exception 45 | * see org.apache.omid.tso.TimestampOracle 46 | * see org.apache.omid.tso.TSOServer 47 | */ 48 | TSOFuture commit(long transactionId, Set writeSet); 49 | 50 | /** 51 | * Closes the communication with the TSO server 52 | * @return nothing. If an error was detected, the future will contain a corresponding protocol exception 53 | */ 54 | TSOFuture close(); 55 | 56 | } 57 | -------------------------------------------------------------------------------- /transaction-client/src/main/java/org/apache/omid/tso/util/DummyCellIdImpl.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.tso.util; 19 | 20 | import org.apache.omid.tso.client.CellId; 21 | 22 | public class DummyCellIdImpl implements CellId { 23 | 24 | private final long cellId; 25 | 26 | public DummyCellIdImpl(long cellId) { 27 | this.cellId = cellId; 28 | } 29 | 30 | @Override 31 | public long getCellId() { 32 | return cellId; 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /transaction-client/src/main/resources/omid-client-config.yml: -------------------------------------------------------------------------------- 1 | # ===================================================================================================================== 2 | # Omid Client Configuration 3 | # ===================================================================================================================== 4 | 5 | # --------------------------------------------------------------------------------------------------------------------- 6 | # Basic connection parameters to TSO Server 7 | # --------------------------------------------------------------------------------------------------------------------- 8 | 9 | # Direct connection to host:port 10 | connectionType: !!org.apache.omid.tso.client.OmidClientConfiguration$ConnType DIRECT 11 | connectionString: "localhost:54758" 12 | 13 | # When Omid is working in High Availability mode, two or more replicas of the TSO server are running in primary/backup 14 | # mode. When a TSO server replica is elected as master, it publishes its address through ZK. In order to configure 15 | # the Omid client to access the TSO server in HA mode: 16 | # 1) set 'connectionType' to !!org.apache.omid.tso.client.OmidClientConfiguration$ConnType HA 17 | # 2) set 'connectionString' to the ZK cluster connection string where the server is publishing its address 18 | zkConnectionTimeoutInSecs: 10 19 | # In HA mode, make sure that the next settings match same settings on the TSO server side 20 | zkNamespace: "omid" 21 | zkCurrentTsoPath: "/current-tso" 22 | 23 | # --------------------------------------------------------------------------------------------------------------------- 24 | # Communication protocol parameters 25 | # --------------------------------------------------------------------------------------------------------------------- 26 | # TODO: describe these parameters 27 | requestMaxRetries: 5 28 | requestTimeoutInMs: 5000 29 | reconnectionDelayInSecs: 10 30 | retryDelayInMs: 1000 31 | executorThreads: 3 32 | 33 | # --------------------------------------------------------------------------------------------------------------------- 34 | # Transaction Manager parameters 35 | # --------------------------------------------------------------------------------------------------------------------- 36 | 37 | # Configure whether the TM performs the post-commit actions for a tx (update shadow cells and clean commit table entry) 38 | # before returning to the control to the client (SYNC) or in parallel (ASYNC) 39 | postCommitMode: !!org.apache.omid.tso.client.OmidClientConfiguration$PostCommitMode SYNC -------------------------------------------------------------------------------- /transaction-client/src/test/java/org/apache/omid/tso/client/TestOmidClientConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.tso.client; 19 | 20 | import org.testng.Assert; 21 | import org.testng.annotations.Test; 22 | 23 | public class TestOmidClientConfiguration { 24 | 25 | @Test(timeOut = 10_000) 26 | public void testYamlReading() { 27 | OmidClientConfiguration configuration = new OmidClientConfiguration(); 28 | Assert.assertNotNull(configuration.getConnectionString()); 29 | Assert.assertNotNull(configuration.getConnectionType()); 30 | } 31 | 32 | } -------------------------------------------------------------------------------- /tso-server/bin/omid-env.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | # Set the flags to pass to the jvm when running omid 20 | # export JVM_FLAGS=-Xmx8096m 21 | # --------------------------------------------------------------------------------------------------------------------- 22 | # Check if HADOOP_CONF_DIR and HBASE_CONF_DIR are set 23 | # --------------------------------------------------------------------------------------------------------------------- 24 | 25 | if [ -z ${HADOOP_CONF_DIR+x} ]; then echo "WARNING: HADOOP_CONF_DIR is unset"; else echo "HADOOP_CONF_DIR is set to '$HADOOP_CONF_DIR'"; fi 26 | if [ -z ${HBASE_CONF_DIR+x} ]; then echo "WARNING: HBASE_CONF_DIR is unset"; else echo "HBASE_CONF_DIR is set to '$HBASE_CONF_DIR'"; fi 27 | 28 | -------------------------------------------------------------------------------- /tso-server/bin/omid.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | SCRIPTDIR=`dirname $0` 21 | cd $SCRIPTDIR; 22 | 23 | # Load Omid environment variables 24 | source omid-env.sh 25 | 26 | # Configure classpath... 27 | CLASSPATH=../conf:${HBASE_CONF_DIR}:${HADOOP_CONF_DIR} 28 | 29 | # ...for source release and... 30 | for j in ../target/omid-tso*.jar; do 31 | CLASSPATH=$CLASSPATH:$j 32 | done 33 | 34 | # and for binary release 35 | for j in ../omid-tso*.jar; do 36 | CLASSPATH=$CLASSPATH:$j 37 | done 38 | for j in ../lib/*.jar; do 39 | CLASSPATH=$CLASSPATH:$j 40 | done 41 | 42 | tso() { 43 | exec java $JVM_FLAGS -cp $CLASSPATH org.apache.omid.tso.TSOServer $@ 44 | } 45 | 46 | tsoRelauncher() { 47 | until ./omid.sh tso $@; do 48 | echo "TSO Server crashed with exit code $?. Re-launching..." >&2 49 | sleep 1 50 | done 51 | } 52 | 53 | createHBaseCommitTable() { 54 | exec java -cp $CLASSPATH org.apache.omid.tools.hbase.OmidTableManager commit-table $@ 55 | } 56 | 57 | createHBaseTimestampTable() { 58 | exec java -cp $CLASSPATH org.apache.omid.tools.hbase.OmidTableManager timestamp-table $@ 59 | } 60 | 61 | usage() { 62 | echo "Usage: omid.sh " 63 | echo "where is one of:" 64 | echo " tso Starts The Status Oracle server (TSO)" 65 | echo " tso-relauncher Starts The Status Oracle server (TSO) re-launching it if the process exits" 66 | echo " create-hbase-commit-table Creates the hbase commit table." 67 | echo " create-hbase-timestamp-table Creates the hbase timestamp table." 68 | } 69 | 70 | # if no args specified, show usage 71 | if [ $# = 0 ]; then 72 | usage; 73 | exit 1 74 | fi 75 | 76 | COMMAND=$1 77 | shift 78 | 79 | if [ "$COMMAND" = "tso" ]; then 80 | tso $@; 81 | elif [ "$COMMAND" = "tso-relauncher" ]; then 82 | tsoRelauncher $@; 83 | elif [ "$COMMAND" = "create-hbase-commit-table" ]; then 84 | createHBaseCommitTable $@; 85 | elif [ "$COMMAND" = "create-hbase-timestamp-table" ]; then 86 | createHBaseTimestampTable $@; 87 | else 88 | exec java -cp $CLASSPATH $COMMAND $@ 89 | fi 90 | 91 | 92 | -------------------------------------------------------------------------------- /tso-server/conf/hbase-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | hbase.zookeeper.quorum 4 | localhost 5 | 6 | 7 | -------------------------------------------------------------------------------- /tso-server/conf/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /tso-server/conf/omid-server-configuration.yml: -------------------------------------------------------------------------------- 1 | # ===================================================================================================================== 2 | # 3 | # Omid TSO Server Configuration 4 | # --------------------------------------------------------------------------------------------------------------------- 5 | # 6 | # Tune here the default values for TSO server config parameters found in 'default-omid-server-configuration.yml' file 7 | # 8 | # ===================================================================================================================== 9 | -------------------------------------------------------------------------------- /tso-server/maven/assembly/bin.xml: -------------------------------------------------------------------------------- 1 | 4 | 5 | bin 6 | 7 | 8 | tar.gz 9 | 10 | 11 | true 12 | 13 | 14 | 15 | 16 | 17 | true 18 | 19 | 20 | org.apache.omid:omid-hbase-commit-table 21 | org.apache.omid:omid-codahale-metrics 22 | org.apache.omid:omid-hbase-tools 23 | 24 | 25 | /lib 26 | false 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | target 35 | / 36 | 37 | ${project.artifactId}-${project.version}.jar 38 | 39 | 40 | 41 | conf 42 | 43 | 44 | bin 45 | 46 | 47 | 644 48 | 49 | ${basedir}/*.txt 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | /lib 60 | false 61 | runtime 62 | false 63 | 64 | 65 | 66 | 67 | -------------------------------------------------------------------------------- /tso-server/src/main/java/org/apache/omid/tso/BatchPoolModule.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.tso; 19 | 20 | import com.google.inject.AbstractModule; 21 | import com.google.inject.Provides; 22 | import org.apache.commons.pool2.ObjectPool; 23 | import org.apache.commons.pool2.impl.GenericObjectPool; 24 | import org.apache.commons.pool2.impl.GenericObjectPoolConfig; 25 | import org.slf4j.Logger; 26 | import org.slf4j.LoggerFactory; 27 | 28 | import javax.inject.Singleton; 29 | import java.util.ArrayList; 30 | import java.util.List; 31 | 32 | public class BatchPoolModule extends AbstractModule { 33 | 34 | private static final Logger LOG = LoggerFactory.getLogger(BatchPoolModule.class); 35 | 36 | private final TSOServerConfig config; 37 | 38 | public BatchPoolModule(TSOServerConfig config) { 39 | this.config = config; 40 | } 41 | 42 | @Override 43 | protected void configure() { 44 | } 45 | 46 | @Provides 47 | @Singleton 48 | ObjectPool getBatchPool() throws Exception { 49 | 50 | int poolSize = config.getNumConcurrentCTWriters(); 51 | int batchSize = config.getBatchSizePerCTWriter(); 52 | 53 | LOG.info("Pool Size (# of Batches) {}; Batch Size {}", poolSize, batchSize); 54 | LOG.info("Total Batch Size (Pool size * Batch Size): {}", poolSize * batchSize); 55 | GenericObjectPoolConfig config = new GenericObjectPoolConfig(); 56 | config.setMaxTotal(poolSize); 57 | config.setBlockWhenExhausted(true); 58 | GenericObjectPool batchPool = new GenericObjectPool<>(new Batch.BatchFactory(batchSize), config); 59 | LOG.info("Pre-creating objects in the pool..."); // TODO There should be a better way to do this 60 | List batches = new ArrayList<>(poolSize); 61 | for (int i = 0; i < poolSize; i++) { 62 | batches.add(batchPool.borrowObject()); 63 | } 64 | for (Batch batch : batches) { 65 | batchPool.returnObject(batch); 66 | } 67 | return batchPool; 68 | 69 | } 70 | 71 | } 72 | -------------------------------------------------------------------------------- /tso-server/src/main/java/org/apache/omid/tso/FatalExceptionHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.tso; 19 | 20 | import com.lmax.disruptor.ExceptionHandler; 21 | import org.slf4j.Logger; 22 | import org.slf4j.LoggerFactory; 23 | 24 | class FatalExceptionHandler implements ExceptionHandler { 25 | 26 | private static final Logger LOG = LoggerFactory.getLogger(FatalExceptionHandler.class); 27 | 28 | Panicker panicker; 29 | 30 | FatalExceptionHandler(Panicker panicker) { 31 | this.panicker = panicker; 32 | } 33 | 34 | @Override 35 | public void handleEventException(Throwable ex, long sequence, Object event) { 36 | 37 | LOG.error("Uncaught exception throws for sequence {}, event {}", sequence, event, ex); 38 | panicker.panic("Uncaught exception in disruptor thread", ex); 39 | 40 | } 41 | 42 | @Override 43 | public void handleOnShutdownException(Throwable ex) { 44 | LOG.warn("Uncaught exception shutting down", ex); 45 | } 46 | 47 | @Override 48 | public void handleOnStartException(Throwable ex) { 49 | panicker.panic("Uncaught exception starting up", ex); 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /tso-server/src/main/java/org/apache/omid/tso/InMemoryCommitTableStorageModule.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.tso; 19 | 20 | import com.google.inject.AbstractModule; 21 | import org.apache.omid.committable.CommitTable; 22 | import org.apache.omid.committable.NullCommitTable; 23 | 24 | import javax.inject.Singleton; 25 | 26 | public class InMemoryCommitTableStorageModule extends AbstractModule { 27 | 28 | @Override 29 | public void configure() { 30 | 31 | bind(CommitTable.class).to(NullCommitTable.class).in(Singleton.class); 32 | 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /tso-server/src/main/java/org/apache/omid/tso/InMemoryTimestampStorageModule.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.tso; 19 | 20 | import com.google.inject.AbstractModule; 21 | import org.apache.omid.timestamp.storage.TimestampStorage; 22 | import org.apache.omid.tso.TimestampOracleImpl.InMemoryTimestampStorage; 23 | 24 | import javax.inject.Singleton; 25 | 26 | public class InMemoryTimestampStorageModule extends AbstractModule { 27 | 28 | @Override 29 | public void configure() { 30 | 31 | bind(TimestampStorage.class).to(InMemoryTimestampStorage.class).in(Singleton.class); 32 | 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /tso-server/src/main/java/org/apache/omid/tso/LeaseManagement.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.tso; 19 | 20 | public interface LeaseManagement { 21 | 22 | class LeaseManagementException extends Exception { 23 | 24 | private static final long serialVersionUID = -2061376444591776881L; 25 | 26 | LeaseManagementException(String msg) { 27 | super(msg); 28 | } 29 | 30 | 31 | LeaseManagementException(String msg, Exception e) { 32 | super(msg, e); 33 | } 34 | 35 | } 36 | 37 | /** 38 | * Allows to start the service implementing the lease management. 39 | */ 40 | void startService() throws Exception; 41 | 42 | /** 43 | * Allows to stop the service implementing the lease management. 44 | */ 45 | void stopService() throws Exception; 46 | 47 | /** 48 | * Check if the instance is still is under the lease period. 49 | */ 50 | boolean stillInLeasePeriod(); 51 | 52 | } 53 | -------------------------------------------------------------------------------- /tso-server/src/main/java/org/apache/omid/tso/LongCache.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.tso; 19 | 20 | public class LongCache { 21 | 22 | private final long[] cache; 23 | private final int size; 24 | private final int associativity; 25 | 26 | public LongCache(int size, int associativity) { 27 | this.size = size; 28 | this.cache = new long[2 * (size + associativity)]; 29 | this.associativity = associativity; 30 | } 31 | 32 | public long set(long key, long value) { 33 | final int index = index(key); 34 | int oldestIndex = 0; 35 | long oldestValue = Long.MAX_VALUE; 36 | for (int i = 0; i < associativity; ++i) { 37 | int currIndex = 2 * (index + i); 38 | if (cache[currIndex] == key) { 39 | oldestValue = 0; 40 | oldestIndex = currIndex; 41 | break; 42 | } 43 | if (cache[currIndex + 1] <= oldestValue) { 44 | oldestValue = cache[currIndex + 1]; 45 | oldestIndex = currIndex; 46 | } 47 | } 48 | cache[oldestIndex] = key; 49 | cache[oldestIndex + 1] = value; 50 | return oldestValue; 51 | } 52 | 53 | public long get(long key) { 54 | final int index = index(key); 55 | for (int i = 0; i < associativity; ++i) { 56 | int currIndex = 2 * (index + i); 57 | if (cache[currIndex] == key) { 58 | return cache[currIndex + 1]; 59 | } 60 | } 61 | return 0; 62 | } 63 | 64 | private int index(long hash) { 65 | return (int) (Math.abs(hash) % size); 66 | } 67 | 68 | } 69 | -------------------------------------------------------------------------------- /tso-server/src/main/java/org/apache/omid/tso/MockPanicker.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.tso; 19 | 20 | import org.slf4j.Logger; 21 | import org.slf4j.LoggerFactory; 22 | 23 | public class MockPanicker implements Panicker { 24 | 25 | private static final Logger LOG = LoggerFactory.getLogger(MockPanicker.class); 26 | 27 | @Override 28 | public void panic(String reason) { 29 | panic(reason, new Throwable("Mock Panicker")); 30 | } 31 | 32 | @Override 33 | public void panic(String reason, Throwable cause) { 34 | LOG.error("PANICKING: {}", reason, cause); 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /tso-server/src/main/java/org/apache/omid/tso/Panicker.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.tso; 19 | 20 | public interface Panicker { 21 | 22 | void panic(String reason); 23 | 24 | void panic(String reason, Throwable cause); 25 | 26 | } 27 | -------------------------------------------------------------------------------- /tso-server/src/main/java/org/apache/omid/tso/PersistenceProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.tso; 19 | 20 | import org.jboss.netty.channel.Channel; 21 | 22 | import java.io.Closeable; 23 | import java.util.concurrent.Future; 24 | 25 | interface PersistenceProcessor extends Closeable { 26 | 27 | void addCommitToBatch(long startTimestamp, long commitTimestamp, Channel c, MonitoringContext monCtx) 28 | throws Exception; 29 | 30 | void addCommitRetryToBatch(long startTimestamp, Channel c, MonitoringContext monCtx) throws Exception; 31 | 32 | void addAbortToBatch(long startTimestamp, Channel c, MonitoringContext monCtx) throws Exception; 33 | 34 | void addTimestampToBatch(long startTimestamp, Channel c, MonitoringContext monCtx) throws Exception; 35 | 36 | void triggerCurrentBatchFlush() throws Exception; 37 | 38 | Future persistLowWatermark(long lowWatermark); 39 | } 40 | -------------------------------------------------------------------------------- /tso-server/src/main/java/org/apache/omid/tso/RequestProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.tso; 19 | 20 | import org.jboss.netty.channel.Channel; 21 | 22 | import java.io.Closeable; 23 | import java.util.Collection; 24 | 25 | // NOTE: public is required explicitly in the interface definition for Guice injection 26 | public interface RequestProcessor extends TSOStateManager.StateObserver, Closeable { 27 | 28 | void timestampRequest(Channel c, MonitoringContext monCtx); 29 | 30 | void commitRequest(long startTimestamp, Collection writeSet, boolean isRetry, Channel c, MonitoringContext monCtx); 31 | 32 | } 33 | -------------------------------------------------------------------------------- /tso-server/src/main/java/org/apache/omid/tso/RetryProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.tso; 19 | 20 | import org.jboss.netty.channel.Channel; 21 | 22 | import java.io.Closeable; 23 | 24 | interface RetryProcessor extends Closeable { 25 | void disambiguateRetryRequestHeuristically(long startTimestamp, Channel c, MonitoringContext monCtx); 26 | } 27 | -------------------------------------------------------------------------------- /tso-server/src/main/java/org/apache/omid/tso/RuntimeExceptionPanicker.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.tso; 19 | 20 | public class RuntimeExceptionPanicker implements Panicker { 21 | 22 | @Override 23 | public void panic(String reason) { 24 | panic(reason, new Throwable("TSO Error")); 25 | } 26 | 27 | @Override 28 | public void panic(String reason, Throwable cause) { 29 | throw new RuntimeException(reason, cause); 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /tso-server/src/main/java/org/apache/omid/tso/SystemExitPanicker.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.tso; 19 | 20 | import org.slf4j.Logger; 21 | import org.slf4j.LoggerFactory; 22 | 23 | public class SystemExitPanicker implements Panicker { 24 | 25 | private static final Logger LOG = LoggerFactory.getLogger(SystemExitPanicker.class); 26 | 27 | @Override 28 | public void panic(String reason) { 29 | panic(reason, new Throwable("TSO Error")); 30 | } 31 | 32 | @Override 33 | public void panic(String reason, Throwable cause) { 34 | LOG.error(reason, cause); 35 | System.exit(-1); 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /tso-server/src/main/java/org/apache/omid/tso/TSOStateManagerImpl.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.tso; 19 | 20 | import com.google.common.base.Preconditions; 21 | import org.slf4j.Logger; 22 | import org.slf4j.LoggerFactory; 23 | 24 | import javax.inject.Inject; 25 | import java.util.ArrayList; 26 | import java.util.List; 27 | 28 | /** 29 | * Implements the management of the state of the TSO 30 | */ 31 | public class TSOStateManagerImpl implements TSOStateManager { 32 | 33 | private static final Logger LOG = LoggerFactory.getLogger(TSOStateManagerImpl.class); 34 | 35 | private List stateObservers = new ArrayList<>(); 36 | 37 | private TSOState state; 38 | 39 | private TimestampOracle timestampOracle; 40 | 41 | @Inject 42 | public TSOStateManagerImpl(TimestampOracle timestampOracle) { 43 | this.timestampOracle = timestampOracle; 44 | } 45 | 46 | @Override 47 | public synchronized void register(StateObserver newObserver) { 48 | Preconditions.checkNotNull(newObserver, "Trying to register a null observer"); 49 | if (!stateObservers.contains(newObserver)) { 50 | stateObservers.add(newObserver); 51 | } 52 | } 53 | 54 | @Override 55 | public synchronized void unregister(StateObserver observer) { 56 | stateObservers.remove(observer); 57 | } 58 | 59 | @Override 60 | public TSOState initialize() throws Exception { 61 | 62 | LOG.info("Initializing TSO Server state..."); 63 | // The timestamp oracle dictates the new state 64 | timestampOracle.initialize(); 65 | long lowWatermark = timestampOracle.getLast(); 66 | // In this implementation the epoch == low watermark 67 | long epoch = lowWatermark; 68 | state = new TSOState(lowWatermark, epoch); 69 | 70 | // Then, notify registered observers about the new state 71 | for (StateObserver stateObserver : stateObservers) { 72 | stateObserver.update(state); 73 | } 74 | LOG.info("TSO Server state {}", state); 75 | return state; 76 | 77 | } 78 | 79 | } 80 | -------------------------------------------------------------------------------- /tso-server/src/main/java/org/apache/omid/tso/TimestampOracle.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.tso; 19 | 20 | import java.io.IOException; 21 | 22 | /** 23 | * Functionality of a service delivering monotonic increasing timestamps. 24 | */ 25 | public interface TimestampOracle { 26 | 27 | /** 28 | * Allows the initialization of the Timestamp Oracle service. 29 | * @throws IOException 30 | * raised if a problem during initialization is shown. 31 | */ 32 | void initialize() throws IOException; 33 | 34 | /** 35 | * Returns the next timestamp. 36 | */ 37 | long next(); 38 | 39 | /** 40 | * Returns the last timestamp assigned. 41 | */ 42 | long getLast(); 43 | 44 | } -------------------------------------------------------------------------------- /tso-server/src/main/java/org/apache/omid/tso/TsoServerDaemon.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.tso; 19 | 20 | import org.apache.commons.daemon.Daemon; 21 | import org.apache.commons.daemon.DaemonContext; 22 | import org.slf4j.Logger; 23 | import org.slf4j.LoggerFactory; 24 | 25 | /** 26 | * Wraps TSO Server as a Daemon 27 | */ 28 | public class TsoServerDaemon implements Daemon { 29 | 30 | private static final Logger LOG = LoggerFactory.getLogger(TsoServerDaemon.class); 31 | private TSOServer tsoServer; 32 | 33 | @Override 34 | public void init(DaemonContext daemonContext) throws Exception { 35 | try { 36 | TSOServerConfig config = new TSOServerConfig(); 37 | LOG.info("Creating TSOServer instance as a Daemon process..."); 38 | tsoServer = TSOServer.getInitializedTsoServer(config); 39 | LOG.info("TSOServer instance for Daemon process created"); 40 | } catch (Exception e) { 41 | LOG.error("Error creating TSOServer instance", e); 42 | throw e; 43 | } 44 | } 45 | 46 | @Override 47 | public void start() throws Exception { 48 | tsoServer.startAndWait(); 49 | } 50 | 51 | @Override 52 | public void stop() throws Exception { 53 | tsoServer.stopAndWait(); 54 | } 55 | 56 | @Override 57 | public void destroy() { 58 | } 59 | 60 | } 61 | -------------------------------------------------------------------------------- /tso-server/src/main/java/org/apache/omid/tso/VoidLeaseManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.tso; 19 | 20 | import java.io.IOException; 21 | 22 | public class VoidLeaseManager implements LeaseManagement { 23 | 24 | private final TSOChannelHandler tsoChannelHandler; 25 | private TSOStateManager stateManager; 26 | 27 | public VoidLeaseManager(TSOChannelHandler tsoChannelHandler, TSOStateManager stateManager) { 28 | this.tsoChannelHandler = tsoChannelHandler; 29 | this.stateManager = stateManager; 30 | } 31 | 32 | @Override 33 | public void startService() throws Exception { 34 | try { 35 | stateManager.initialize(); 36 | tsoChannelHandler.reconnect(); 37 | } catch (IOException e) { 38 | throw new LeaseManagementException("Error initializing Lease Manager", e); 39 | } 40 | } 41 | 42 | @Override 43 | public void stopService() throws Exception { 44 | tsoChannelHandler.closeConnection(); 45 | } 46 | 47 | @Override 48 | public boolean stillInLeasePeriod() { 49 | // We should always return true 50 | return true; 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /tso-server/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=WARN,console 2 | 3 | # Logging Threshold 4 | log4j.threshold=ALL 5 | 6 | # 7 | # Daily Rolling File Appender 8 | # 9 | log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender 10 | log4j.appender.DRFA.File=${omid.log.dir}/${omid.log.file} 11 | 12 | # Rollver at midnight 13 | log4j.appender.DRFA.DatePattern=.yyyy-MM-dd 14 | 15 | # 30-day backup 16 | #log4j.appender.DRFA.MaxBackupIndex=30 17 | log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout 18 | 19 | # Pattern format: Date LogLevel LoggerName LogMessage 20 | log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 21 | 22 | # Debugging Pattern format 23 | #log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n 24 | 25 | 26 | # 27 | # console 28 | # Add "console" to rootlogger above if you want to use this 29 | # 30 | log4j.appender.console=org.apache.log4j.ConsoleAppender 31 | log4j.appender.console.target=System.err 32 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 33 | log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} [%t] %p %c{2}: %m%n 34 | 35 | # Custom Logging levels 36 | log4j.logger.org.apache.zookeeper=ERROR 37 | log4j.logger.org.apache.bookkeeper=ERROR 38 | log4j.logger.org.apache.hadoop.hbase=ERROR 39 | log4j.logger.org.apache.hadoop.ipc=ERROR 40 | log4j.logger.org.apache.omid=INFO 41 | #log4j.logger.org.apache.omid.regionserver.TransactionalRegionServer=TRACE 42 | #log4j.logger.org.apache.omid.TestBasicTransaction=TRACE 43 | #log4j.logger.org.apache.omid.client.TSOClient=TRACE 44 | #log4j.logger.org.apache.omid.client.TransactionState=TRACE 45 | #log4j.logger.org.apache.omid.tso.ThroughputMonitor=TRACE 46 | #log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG 47 | 48 | # Make these two classes INFO-level. Make them DEBUG to see more zk debug. 49 | #log4j.logger.org.apache.hadoop.dfs=DEBUG 50 | # Set this class to log INFO only otherwise its OTT 51 | 52 | # Uncomment the below if you want to remove logging of client region caching' 53 | # and scan of .META. messages 54 | # log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=INFO 55 | # log4j.logger.org.apache.hadoop.hbase.client.MetaScanner=INFO 56 | -------------------------------------------------------------------------------- /tso-server/src/test/java/org/apache/omid/tso/PausableTimestampOracle.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.tso; 19 | 20 | import org.apache.omid.metrics.MetricsRegistry; 21 | import org.apache.omid.timestamp.storage.TimestampStorage; 22 | import org.slf4j.Logger; 23 | import org.slf4j.LoggerFactory; 24 | 25 | import javax.inject.Inject; 26 | import java.io.IOException; 27 | 28 | public class PausableTimestampOracle extends TimestampOracleImpl { 29 | 30 | private static final Logger LOG = LoggerFactory.getLogger(PausableTimestampOracle.class); 31 | 32 | private volatile boolean tsoPaused = false; 33 | 34 | @Inject 35 | public PausableTimestampOracle(MetricsRegistry metrics, 36 | TimestampStorage tsStorage, 37 | Panicker panicker) throws IOException { 38 | super(metrics, tsStorage, panicker); 39 | } 40 | 41 | @Override 42 | public long next() { 43 | while (tsoPaused) { 44 | synchronized (this) { 45 | try { 46 | this.wait(); 47 | } catch (InterruptedException e) { 48 | LOG.error("Interrupted whilst paused"); 49 | Thread.currentThread().interrupt(); 50 | } 51 | } 52 | } 53 | return super.next(); 54 | } 55 | 56 | public synchronized void pause() { 57 | tsoPaused = true; 58 | this.notifyAll(); 59 | } 60 | 61 | public synchronized void resume() { 62 | tsoPaused = false; 63 | this.notifyAll(); 64 | } 65 | 66 | public boolean isTSOPaused() { 67 | return tsoPaused; 68 | } 69 | 70 | } 71 | -------------------------------------------------------------------------------- /tso-server/src/test/java/org/apache/omid/tso/TSOServerConfigTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.tso; 19 | 20 | import org.testng.annotations.Test; 21 | 22 | public class TSOServerConfigTest { 23 | 24 | @Test(timeOut = 10_000) 25 | public void testParsesOK() throws Exception { 26 | new TSOServerConfig("test-omid.yml"); 27 | } 28 | 29 | } -------------------------------------------------------------------------------- /tso-server/src/test/java/org/apache/omid/tso/client/TSOClientAccessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.tso.client; 19 | 20 | import org.apache.statemachine.StateMachine.FsmImpl; 21 | 22 | public class TSOClientAccessor { 23 | 24 | public static void closeChannel(TSOClient tsoClient) throws InterruptedException { 25 | FsmImpl fsm = (FsmImpl) tsoClient.fsm; 26 | TSOClient.ConnectedState connectedState = (TSOClient.ConnectedState) fsm.getState(); 27 | connectedState.channel.close().await(); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /tso-server/src/test/java/org/apache/omid/tso/client/TSOClientOneShot.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.omid.tso.client; 19 | 20 | import org.apache.omid.proto.TSOProto; 21 | import org.apache.omid.proto.TSOProto.Response; 22 | import org.slf4j.Logger; 23 | import org.slf4j.LoggerFactory; 24 | 25 | import java.util.concurrent.ExecutionException; 26 | 27 | /** 28 | * Communication endpoint for TSO clients. 29 | */ 30 | public class TSOClientOneShot { 31 | 32 | private static final Logger LOG = LoggerFactory.getLogger(TSOClientOneShot.class); 33 | 34 | private final String host; 35 | private final int port; 36 | 37 | public TSOClientOneShot(String host, int port) { 38 | 39 | this.host = host; 40 | this.port = port; 41 | 42 | } 43 | 44 | public TSOProto.Response makeRequest(TSOProto.Request request) 45 | throws InterruptedException, ExecutionException { 46 | TSOClientRaw raw = new TSOClientRaw(host, port); 47 | 48 | // do handshake 49 | TSOProto.HandshakeRequest.Builder handshake = TSOProto.HandshakeRequest.newBuilder(); 50 | handshake.setClientCapabilities(TSOProto.Capabilities.newBuilder().build()); 51 | raw.write(TSOProto.Request.newBuilder() 52 | .setHandshakeRequest(handshake.build()).build()); 53 | Response response = raw.getResponse().get(); 54 | assert (response.getHandshakeResponse().getClientCompatible()); 55 | 56 | raw.write(request); 57 | response = raw.getResponse().get(); 58 | 59 | raw.close(); 60 | return response; 61 | } 62 | 63 | } 64 | -------------------------------------------------------------------------------- /tso-server/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO,console 2 | 3 | # Logging Threshold 4 | #log4j.threshold=INFO 5 | 6 | # 7 | # Daily Rolling File Appender 8 | # 9 | log4j.appender.DRFA.threshold=TRACE 10 | log4j.appender.DRFA=org.apache.log4j.FileAppender 11 | log4j.appender.DRFA.File=omid.log 12 | 13 | # Rollver at midnight 14 | #log4j.appender.DRFA.DatePattern=.yyyy-MM-dd 15 | log4j.appender.DRFA.append=false 16 | 17 | # 30-day backup 18 | #log4j.appender.DRFA.MaxBackupIndex=30 19 | log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout 20 | 21 | # Pattern format: Date LogLevel LoggerName LogMessage 22 | log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 23 | 24 | # Debugging Pattern format 25 | #log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n 26 | 27 | 28 | # 29 | # console 30 | # Add "console" to rootlogger above if you want to use this 31 | # 32 | log4j.appender.console=org.apache.log4j.ConsoleAppender 33 | log4j.appender.console.threshold=DEBUG 34 | log4j.appender.console.target=System.err 35 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 36 | log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} [%t] %p %c{2}: %m%n 37 | 38 | # Custom Logging levels 39 | log4j.logger.org.apache.zookeeper=ERROR 40 | log4j.logger.org.apache.bookkeeper=FATAL 41 | log4j.logger.org.apache.hadoop.hbase=ERROR 42 | log4j.logger.org.apache.hadoop.ipc=ERROR 43 | log4j.logger.org.apache.omid=INFO 44 | #log4j.logger.org.apache.omid.regionserver.TransactionalRegionServer=TRACE 45 | #log4j.logger.org.apache.omid.TestBasicTransaction=TRACE 46 | #log4j.logger.org.apache.omid.client.TSOClient=TRACE 47 | #log4j.logger.org.apache.omid.client.TransactionState=TRACE 48 | #log4j.logger.org.apache.omid.OmidTestBase=TRACE 49 | #log4j.logger.org.apache.omid.tso.ThroughputMonitor=INFO 50 | #log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG 51 | 52 | # Make these two classes INFO-level. Make them DEBUG to see more zk debug. 53 | #log4j.logger.org.apache.hadoop.dfs=DEBUG 54 | # Set this class to log INFO only otherwise its OTT 55 | 56 | # Uncomment the below if you want to remove logging of client region caching' 57 | # and scan of .META. messages 58 | # log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=INFO 59 | # log4j.logger.org.apache.hadoop.hbase.client.MetaScanner=INFO 60 | -------------------------------------------------------------------------------- /tso-server/src/test/resources/test-omid.yml: -------------------------------------------------------------------------------- 1 | # Non High Availability configuration with HBase for everything 2 | 3 | port: 54758 4 | conflictMapSize: 100000000 5 | numConcurrentCTWriters: 2 6 | batchSizePerCTWriter: 500 7 | batchPersistTimeoutInMs: 100 8 | networkIfaceName: eth1 9 | 10 | commitTableStoreModule: !!org.apache.omid.committable.hbase.DefaultHBaseCommitTableStorageModule 11 | tableName: "sieve_omid:OMID_TIMESTAMP_F" 12 | 13 | timestampStoreModule: !!org.apache.omid.timestamp.storage.DefaultHBaseTimestampStorageModule 14 | tableName: "sieve_omid:OMID_COMMIT_TABLE_F" 15 | familyName: "MAX_TIMESTAMP_F" 16 | 17 | leaseModule: !!org.apache.omid.tso.VoidLeaseManagementModule [ ] 18 | 19 | metrics: !!org.apache.omid.metrics.NullMetricsProvider [ ] --------------------------------------------------------------------------------