├── .gitignore ├── LICENSE ├── README.rst ├── bin ├── doprotoc.sh ├── dothrift.sh └── jdiff-hbase.sh ├── ch03 ├── bin │ └── run.sh ├── pom.xml └── src │ └── main │ ├── java │ └── client │ │ ├── AppendExample.java │ │ ├── BatchCallbackExample.java │ │ ├── BatchExample.java │ │ ├── BatchSameRowExample.java │ │ ├── BufferedMutatorExample.java │ │ ├── CRUDExample.java │ │ ├── CRUDExamplePreV1API.java │ │ ├── CellComparatorExample.java │ │ ├── CellScannerExample.java │ │ ├── CheckAndDeleteExample.java │ │ ├── CheckAndMutateExample.java │ │ ├── CheckAndPutExample.java │ │ ├── DeleteExample.java │ │ ├── DeleteListErrorExample.java │ │ ├── DeleteListExample.java │ │ ├── FingerprintExample.java │ │ ├── GetCheckExistenceExample.java │ │ ├── GetCloneExample.java │ │ ├── GetClosestRowBeforeExample.java │ │ ├── GetExample.java │ │ ├── GetFluentExample.java │ │ ├── GetListErrorExample.java │ │ ├── GetListExample.java │ │ ├── GetMaxResultsRowOffsetExample1.java │ │ ├── GetMaxResultsRowOffsetExample2.java │ │ ├── GetTryWithResourcesExample.java │ │ ├── MissingRegionExample.java │ │ ├── MutateRowExample.java │ │ ├── PutExample.java │ │ ├── PutIdenticalExample.java │ │ ├── PutListErrorExample1.java │ │ ├── PutListErrorExample2.java │ │ ├── PutListErrorExample3.java │ │ ├── PutListExample.java │ │ ├── PutWriteBufferExample1.java │ │ ├── PutWriteBufferExample2.java │ │ ├── ResultExample.java │ │ ├── RowKeyExample.java │ │ ├── ScanCacheBatchExample.java │ │ ├── ScanExample.java │ │ ├── ScanSlicingExample.java │ │ └── ScanTimeoutExample.java │ └── resources │ └── log4j.properties ├── ch04 ├── bin │ └── run.sh ├── pom.xml └── src │ └── main │ ├── java │ ├── client │ │ ├── IncrementMultipleExample.java │ │ └── IncrementSingleExample.java │ ├── coprocessor │ │ ├── DuplicateRegionObserverExample.java │ │ ├── EndpointBatchExample.java │ │ ├── EndpointCombinedExample.java │ │ ├── EndpointExample.java │ │ ├── EndpointProxyExample.java │ │ ├── LoadWithTableDescriptorExample.java │ │ ├── LoadWithTableDescriptorExample2.java │ │ ├── MasterObserverExample.java │ │ ├── ObserverStatisticsEndpoint.java │ │ ├── ObserverStatisticsExample.java │ │ ├── RegionObserverExample.java │ │ ├── RegionObserverWithBypassExample.java │ │ ├── RegionObserverWithCompleteExample.java │ │ ├── RegionServerObserverExample.java │ │ ├── RowCountEndpoint.java │ │ └── generated │ │ │ ├── ObserverStatisticsProtos.java │ │ │ └── RowCounterProtos.java │ └── filters │ │ ├── ColumnPaginationFilterExample.java │ │ ├── ColumnPrefixFilterExample.java │ │ ├── ColumnRangeFilterExample.java │ │ ├── CustomFilter.java │ │ ├── CustomFilterExample.java │ │ ├── DependentColumnFilterExample.java │ │ ├── FamilyFilterExample.java │ │ ├── FilterListExample.java │ │ ├── FirstKeyOnlyFilterExample.java │ │ ├── FirstKeyValueMatchingQualifiersFilterExample.java │ │ ├── FuzzyRowFilterExample.java │ │ ├── InclusiveStopFilterExample.java │ │ ├── KeyOnlyFilterExample.java │ │ ├── MultiRowRangeFilterExample.java │ │ ├── MultipleColumnPrefixFilterExample.java │ │ ├── PageFilterExample.java │ │ ├── PrefixFilterExample.java │ │ ├── QualifierFilterExample.java │ │ ├── RandomRowFilterExample.java │ │ ├── RowFilterExample.java │ │ ├── SingleColumnValueFilterExample.java │ │ ├── SkipFilterExample.java │ │ ├── TimestampFilterExample.java │ │ ├── ValueFilterExample.java │ │ ├── ValueNotNullFilterExample.java │ │ ├── WhileMatchFilterExample.java │ │ └── generated │ │ └── FilterProtos.java │ ├── protobuf │ ├── CustomFilters.proto │ ├── ObserverStatisticsService.proto │ └── RowCountService.proto │ └── resources │ └── log4j.properties ├── ch05 ├── bin │ └── run.sh ├── pom.xml └── src │ └── main │ ├── java │ ├── admin │ │ ├── ClusterOperationExample.java │ │ ├── ClusterStatusExample.java │ │ ├── CreateTableExample.java │ │ ├── CreateTableWithNamespaceExample.java │ │ ├── CreateTableWithRegionsExample.java │ │ ├── HColumnDescriptorExample.java │ │ ├── ListTablesExample.java │ │ ├── ListTablesExample2.java │ │ ├── ListTablesExample3.java │ │ ├── ModifyTableExample.java │ │ ├── NamespaceDescriptorExample.java │ │ ├── NamespaceExample.java │ │ ├── QuotaExample.java │ │ ├── ServerAndRegionNameExample.java │ │ ├── SnapshotExample.java │ │ ├── TableNameExample.java │ │ └── TableOperationsExample.java │ └── coprocessor │ │ ├── DelayRegionCloseObserver.java │ │ ├── DelayingMasterObserver.java │ │ └── SequentialIdGeneratorObserver.java │ └── resources │ └── log4j.properties ├── ch06 ├── pom.xml └── src │ └── main │ └── java │ ├── org │ └── apache │ │ └── hadoop │ │ └── hbase │ │ ├── thrift │ │ └── generated │ │ │ ├── AlreadyExists.java │ │ │ ├── BatchMutation.java │ │ │ ├── ColumnDescriptor.java │ │ │ ├── Hbase.java │ │ │ ├── IOError.java │ │ │ ├── IllegalArgument.java │ │ │ ├── Mutation.java │ │ │ ├── TAppend.java │ │ │ ├── TCell.java │ │ │ ├── TColumn.java │ │ │ ├── TIncrement.java │ │ │ ├── TRegionInfo.java │ │ │ ├── TRowResult.java │ │ │ └── TScan.java │ │ └── thrift2 │ │ └── generated │ │ ├── TAppend.java │ │ ├── TAuthorization.java │ │ ├── TCellVisibility.java │ │ ├── TColumn.java │ │ ├── TColumnIncrement.java │ │ ├── TColumnValue.java │ │ ├── TDelete.java │ │ ├── TDeleteType.java │ │ ├── TDurability.java │ │ ├── TGet.java │ │ ├── THBaseService.java │ │ ├── TIOError.java │ │ ├── TIllegalArgument.java │ │ ├── TIncrement.java │ │ ├── TMutation.java │ │ ├── TPut.java │ │ ├── TResult.java │ │ ├── TRowMutations.java │ │ ├── TScan.java │ │ └── TTimeRange.java │ ├── pig │ └── JsonStringToMap.java │ ├── rest │ └── RestExample.java │ └── thrift │ └── ThriftExample.java ├── ch07 ├── pom.xml ├── src │ └── main │ │ ├── assembly │ │ └── job.xml │ │ └── java │ │ └── mapreduce │ │ ├── AnalyzeData.java │ │ ├── AnalyzeSnapshotData.java │ │ ├── BytesSplit.java │ │ ├── Driver.java │ │ ├── ImportFromFile.java │ │ ├── ImportFromFile2.java │ │ ├── ImportJsonFromFile.java │ │ ├── InvalidReducerOverride.java │ │ ├── ParseJson.java │ │ ├── ParseJson2.java │ │ └── ParseJsonMulti.java └── test-data.txt ├── ch08 ├── bin │ ├── doprotoc.sh │ └── run.sh ├── pom.xml └── src │ └── main │ ├── java │ ├── client │ │ ├── ScanConsistencyExample1.java │ │ ├── ScanConsistencyExample2.java │ │ └── ScanConsistencyExample3.java │ └── coprocessor │ │ ├── ScanControlObserverEndpoint.java │ │ └── generated │ │ └── ScanControlProtos.java │ └── protobuf │ └── ScanControlService.proto ├── ch09 ├── bin │ └── run.sh ├── pom.xml └── src │ └── main │ ├── java │ ├── client │ │ └── DeleteTimestampExample.java │ └── transactions │ │ └── MultiRowMutationExample.java │ └── resources │ └── log4j.properties ├── ch11 ├── pom.xml └── src │ └── main │ └── java │ └── bulkimport │ └── BulkImportJobExample.java ├── ch12 ├── pom.xml └── src │ └── main │ ├── java │ ├── datatasks │ │ └── RenameTableExample.java │ └── htrace │ │ └── HTraceExample.java │ └── resources │ └── log4j.properties ├── ch13 ├── bin │ └── run.sh ├── pom.xml └── src │ └── main │ └── java │ ├── org │ └── apache │ │ └── hadoop │ │ └── hbase │ │ ├── thrift │ │ └── generated │ │ │ ├── AlreadyExists.java │ │ │ ├── BatchMutation.java │ │ │ ├── ColumnDescriptor.java │ │ │ ├── Hbase.java │ │ │ ├── IOError.java │ │ │ ├── IllegalArgument.java │ │ │ ├── Mutation.java │ │ │ ├── TAppend.java │ │ │ ├── TCell.java │ │ │ ├── TColumn.java │ │ │ ├── TIncrement.java │ │ │ ├── TRegionInfo.java │ │ │ ├── TRowResult.java │ │ │ └── TScan.java │ │ └── thrift2 │ │ └── generated │ │ ├── TAppend.java │ │ ├── TAuthorization.java │ │ ├── TCellVisibility.java │ │ ├── TColumn.java │ │ ├── TColumnIncrement.java │ │ ├── TColumnValue.java │ │ ├── TDelete.java │ │ ├── TDeleteType.java │ │ ├── TDurability.java │ │ ├── TGet.java │ │ ├── THBaseService.java │ │ ├── TIOError.java │ │ ├── TIllegalArgument.java │ │ ├── TIncrement.java │ │ ├── TMutation.java │ │ ├── TPut.java │ │ ├── TResult.java │ │ ├── TRowMutations.java │ │ ├── TScan.java │ │ └── TTimeRange.java │ ├── security │ ├── AccessControlExample.java │ ├── AuthenticatedUser.java │ ├── TokenExample.java │ └── VisibilityLabelExample.java │ └── thrift │ └── SecureThriftExample.java ├── cluster ├── Vagrantfile └── ansible │ ├── README.md │ ├── ansible.cfg │ ├── clean.yml │ ├── group_vars │ ├── all │ ├── hbase_backup_masters │ ├── hbase_master │ ├── hbase_workers │ ├── kdc-server │ └── zookeepers │ ├── host_vars │ ├── master-1 │ ├── master-2 │ └── master-3 │ ├── install.yml │ ├── inventories │ ├── cluster.inv │ └── hbasebook.inv │ ├── roles │ ├── common │ │ ├── files │ │ │ ├── etc │ │ │ │ └── init.d │ │ │ │ │ ├── flume-ng-agent │ │ │ │ │ ├── hadoop-hdfs-datanode │ │ │ │ │ ├── hadoop-hdfs-journalnode │ │ │ │ │ ├── hadoop-hdfs-namenode │ │ │ │ │ ├── hadoop-hdfs-secondarynamenode │ │ │ │ │ ├── hadoop-httpfs │ │ │ │ │ ├── hadoop-mapreduce-historyserver │ │ │ │ │ ├── hadoop-yarn-nodemanager │ │ │ │ │ ├── hadoop-yarn-proxyserver │ │ │ │ │ ├── hadoop-yarn-resourcemanager │ │ │ │ │ ├── hbase-master │ │ │ │ │ ├── hbase-regionserver │ │ │ │ │ ├── hbase-rest │ │ │ │ │ ├── hbase-solr-indexer │ │ │ │ │ ├── hbase-thrift │ │ │ │ │ ├── hive-metastore │ │ │ │ │ ├── hive-server2 │ │ │ │ │ ├── hue │ │ │ │ │ ├── impala-catalog │ │ │ │ │ ├── impala-server │ │ │ │ │ ├── impala-state-store │ │ │ │ │ ├── oozie │ │ │ │ │ ├── sentry-store │ │ │ │ │ ├── solr-server │ │ │ │ │ ├── spark-history-server │ │ │ │ │ ├── spark-master │ │ │ │ │ ├── spark-worker │ │ │ │ │ ├── sqoop-metastore │ │ │ │ │ ├── sqoop2-server │ │ │ │ │ └── zookeeper-server │ │ │ └── hosts │ │ ├── handlers │ │ │ └── main.yml │ │ └── tasks │ │ │ └── main.yml │ ├── hadoop │ │ ├── tasks │ │ │ └── main.yml │ │ └── templates │ │ │ └── etc │ │ │ ├── hadoop │ │ │ └── conf │ │ │ │ ├── container-executor.cfg.j2 │ │ │ │ ├── core-site.xml.j2 │ │ │ │ ├── hadoop-env.sh.j2 │ │ │ │ ├── hadoop-policy.xml │ │ │ │ ├── hdfs-site.xml.j2 │ │ │ │ ├── httpfs-env.sh │ │ │ │ ├── httpfs-site.xml │ │ │ │ ├── log4j.properties │ │ │ │ ├── mapred-env.sh.j2 │ │ │ │ ├── mapred-site.xml.j2 │ │ │ │ ├── ssl-client.xml.j2 │ │ │ │ ├── ssl-server.xml.j2 │ │ │ │ ├── yarn-env.sh.j2 │ │ │ │ └── yarn-site.xml.j2 │ │ │ └── systemd │ │ │ └── system │ │ │ ├── datanode.service.j2 │ │ │ ├── historyserver.service.j2 │ │ │ ├── namenode.service.j2 │ │ │ ├── nodemanager.service.j2 │ │ │ └── resourcemanager.service.j2 │ ├── hbase-common │ │ ├── tasks │ │ │ ├── create-hbase-root.yml │ │ │ └── main.yml │ │ └── templates │ │ │ └── etc │ │ │ └── hbase │ │ │ └── conf │ │ │ ├── backup-masters.j2 │ │ │ ├── hadoop-metrics2-hbase.properties.j2 │ │ │ ├── hbase-env.sh.j2 │ │ │ ├── hbase-jaas.conf.j2 │ │ │ ├── hbase-policy.xml.j2 │ │ │ ├── hbase-site.xml.j2 │ │ │ ├── log4j.properties.j2 │ │ │ └── regionservers.j2 │ ├── hbase-master │ │ ├── tasks │ │ │ └── main.yml │ │ └── templates │ │ │ └── etc │ │ │ └── systemd │ │ │ └── system │ │ │ └── hbase-master.service.j2 │ ├── hbase-regionserver │ │ ├── tasks │ │ │ └── main.yml │ │ └── templates │ │ │ └── etc │ │ │ └── systemd │ │ │ └── system │ │ │ └── hbase-regionserver.service.j2 │ ├── hbase-rest │ │ ├── tasks │ │ │ └── main.yml │ │ └── templates │ │ │ └── etc │ │ │ └── systemd │ │ │ └── system │ │ │ └── hbase-rest.service.j2 │ ├── hbase-thrift │ │ ├── tasks │ │ │ └── main.yml │ │ └── templates │ │ │ └── etc │ │ │ └── systemd │ │ │ └── system │ │ │ └── hbase-thrift.service.j2 │ ├── kdc-server │ │ ├── files │ │ │ └── etc │ │ │ │ └── systemd │ │ │ │ └── system │ │ │ │ └── rngd.service │ │ ├── handlers │ │ │ └── main.yml │ │ ├── tasks │ │ │ ├── createkdb5.yml │ │ │ ├── createprincs.yml │ │ │ └── main.yml │ │ └── templates │ │ │ ├── etc │ │ │ └── krb5.conf.j2 │ │ │ └── var │ │ │ └── kerberos │ │ │ └── krb5kdc │ │ │ ├── .k5.INTERNAL.LARSGEORGE.COM │ │ │ ├── kadm5.acl.j2 │ │ │ └── kdc.conf.j2 │ ├── krb-client │ │ ├── files │ │ │ ├── UnlimitedJCEPolicy │ │ │ │ ├── README.txt │ │ │ │ ├── US_export_policy.jar │ │ │ │ └── local_policy.jar │ │ │ └── certs │ │ │ │ ├── rootCA.key │ │ │ │ └── rootCA.pem │ │ ├── handlers │ │ │ └── main.yml │ │ ├── tasks │ │ │ └── main.yml │ │ └── templates │ │ │ └── etc │ │ │ └── krb5.conf.j2 │ ├── monitoring-aggregation │ │ ├── handlers │ │ │ └── main.yml │ │ ├── tasks │ │ │ └── main.yml │ │ └── templates │ │ │ └── etc │ │ │ └── ganglia │ │ │ └── gmetad.conf.j2 │ ├── monitoring-collection │ │ ├── handlers │ │ │ └── main.yml │ │ ├── tasks │ │ │ └── main.yml │ │ └── templates │ │ │ └── etc │ │ │ └── ganglia │ │ │ └── gmond.conf.j2 │ ├── monitoring-frontend │ │ ├── handlers │ │ │ └── main.yml │ │ ├── tasks │ │ │ └── main.yml │ │ └── templates │ │ │ └── etc │ │ │ └── httpd │ │ │ └── conf.d │ │ │ └── ganglia.conf.j2 │ ├── ssl-phase-1 │ │ └── tasks │ │ │ └── main.yml │ ├── ssl-phase-2 │ │ └── tasks │ │ │ └── main.yml │ ├── ssl-phase-3 │ │ └── tasks │ │ │ └── main.yml │ └── zookeeper │ │ ├── tasks │ │ └── main.yml │ │ └── templates │ │ ├── data │ │ └── zookeeper │ │ │ └── myid.j2 │ │ └── etc │ │ ├── systemd │ │ └── system │ │ │ └── zookeeper.service.j2 │ │ └── zookeeper │ │ ├── jaas.conf.j2 │ │ ├── java.env.j2 │ │ ├── log4j.properties.j2 │ │ └── zoo.cfg.j2 │ ├── services.yml │ └── sslcerts.yml ├── common ├── pom.xml └── src │ └── main │ └── java │ └── util │ └── HBaseHelper.java ├── hush ├── .gitignore ├── .settings │ ├── org.eclipse.jdt.core.prefs │ └── org.eclipse.jdt.ui.prefs ├── bin │ ├── hush-config.sh │ └── start-hush.sh ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ ├── hbasebook │ │ └── hush │ │ │ ├── Counters.java │ │ │ ├── DomainManager.java │ │ │ ├── HushMain.java │ │ │ ├── HushUtil.java │ │ │ ├── ResourceManager.java │ │ │ ├── UrlManager.java │ │ │ ├── UserManager.java │ │ │ ├── coprocessor │ │ │ ├── FetchUrlInfoObserver.java │ │ │ └── FetchUrlInfoProtocol.java │ │ │ ├── model │ │ │ ├── ColumnQualifier.java │ │ │ ├── Counter.java │ │ │ ├── LongUrl.java │ │ │ ├── ShortDomain.java │ │ │ ├── ShortUrl.java │ │ │ ├── ShortUrlStatistics.java │ │ │ ├── StatisticsCategory.java │ │ │ ├── TimeFrame.java │ │ │ └── User.java │ │ │ ├── schema │ │ │ └── SchemaManager.java │ │ │ ├── servlet │ │ │ ├── RequestInfo.java │ │ │ ├── filter │ │ │ │ └── RedirectFilter.java │ │ │ └── security │ │ │ │ └── HBaseLoginService.java │ │ │ └── table │ │ │ ├── HushTable.java │ │ │ ├── LongDomainTable.java │ │ │ ├── LongUrlTable.java │ │ │ ├── ShortDomainTable.java │ │ │ ├── ShortUrlTable.java │ │ │ ├── UserShortUrlTable.java │ │ │ └── UserTable.java │ │ └── maxmind │ │ └── geoip │ │ ├── Country.java │ │ ├── DatabaseInfo.java │ │ ├── Location.java │ │ ├── LookupService.java │ │ ├── Region.java │ │ ├── regionName.java │ │ └── timeZone.java │ ├── resources │ ├── GeoIP.dat │ ├── log4j.properties │ └── schema.xml │ └── webapp │ ├── WEB-INF │ └── web.xml │ ├── admin │ ├── domains.jsp │ ├── index.jsp │ └── users.jsp │ ├── details.jsp │ ├── error.jsp │ ├── include │ ├── adminMenu.jsp │ ├── error.jsp │ ├── footer.jsp │ ├── header.jsp │ └── userstats.jsp │ ├── index.jsp │ ├── login.jsp │ ├── logout.jsp │ ├── signup.jsp │ ├── style.css │ └── user │ └── index.jsp └── pom.xml /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.iml 3 | target/ 4 | .idea/ 5 | /cluster/ansible/files/ 6 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | =========================== 2 | HBase: The Definitive Guide 3 | =========================== 4 | 5 | This repository has all the code as used in the HBase book. 6 | 7 | -------------------------------------------------------------------------------- /bin/doprotoc.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Runs the Google Protocol Buffer compiler. 4 | # 5 | # Requirements: protoc 6 | # 7 | # Note: 8 | # - For MacOS with Homebrew run: $ brew install protobuf 9 | # 10 | set -e 11 | 12 | PROTO_BASE_DIR="ch04/src/main/protobuf" 13 | 14 | PROTO_ROWCOUNT="$PROTO_BASE_DIR/RowCountService.proto" 15 | PROTO_FILTERS="$PROTO_BASE_DIR/CustomFilters.proto" 16 | PROTO_OBSERVERSTATS="$PROTO_BASE_DIR/ObserverStatisticsService.proto" 17 | 18 | PROTOS="$PROTO_ROWCOUNT $PROTO_FILTERS $PROTO_OBSERVERSTATS" 19 | 20 | PATH_GENERATED_BASE="ch04/src/main/java" 21 | PATH_GENERATED_COPROS="$PATH_GENERATED_BASE/coprocessor/generated" 22 | PATH_GENERATED_FILTERS="$PATH_GENERATED_BASE/filters/generated" 23 | 24 | PATHS="$PATH_GENERATED_COPROS $PATH_GENERATED_FILTERS" 25 | 26 | # check all is well 27 | if [ ! -d "$PROTO_BASE_DIR" ]; then 28 | echo "Error: this script must run in the project root directort... exiting!" 29 | exit -1 30 | fi 31 | 32 | # check if output directory exists, if not create it 33 | for loc in $PATHS; do 34 | if [ ! -d "$loc" ]; then 35 | echo "creating directory: $loc" 36 | mkdir "$loc" 37 | fi 38 | done 39 | 40 | # run protocol buffer compiler 41 | for proto in $PROTOS; do 42 | if [ -f "$proto" ]; then 43 | echo "compiling protocol: $proto" 44 | protoc -I$PROTO_BASE_DIR --java_out=$PATH_GENERATED_BASE $proto 45 | fi 46 | done 47 | 48 | echo "done." 49 | -------------------------------------------------------------------------------- /bin/dothrift.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Runs the Thrift compiler. 4 | # 5 | # Requirements: 6 | # - Apache Thrift's thrift compiler 7 | # - HBase Thrift file 8 | # 9 | # Usage: $ dothrift.sh 10 | # 11 | # Note: 12 | # - For MacOS with Homebrew run: $ brew install thrift 13 | # 14 | set -e 15 | 16 | PATH_GENERATED="ch06/src/main/java" 17 | THRIFT_FILE="$1" 18 | 19 | # check all is well 20 | if [ $# -eq 0 ]; then 21 | echo "Missing thrift file parameter!" 22 | echo "Usage: $0 " 23 | exit -1 24 | fi 25 | 26 | # run thrift compiler 27 | echo "compiling thrift: $THRIFT_FILE" 28 | thrift -out $PATH_GENERATED --gen java $THRIFT_FILE 29 | 30 | echo "done." 31 | -------------------------------------------------------------------------------- /ch03/bin/run.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | # 3 | # run.sh: helps executing the examples by setting up the Java CLASSPATH. 4 | # 5 | 6 | # get the current directory 7 | bin=`dirname "$0"` 8 | bin=`cd "$bin">/dev/null; pwd` 9 | 10 | # check command line args 11 | if [[ $# == 0 ]]; then 12 | echo "usage: $(basename $0) " 13 | exit 1; 14 | fi 15 | 16 | # set up Maven environment 17 | MVN="mvn" 18 | if [ "$MAVEN_HOME" != "" ]; then 19 | MVN=${MAVEN_HOME}/bin/mvn 20 | fi 21 | 22 | # classpath initially contains $HBASE_CONF_DIR 23 | CLASSPATH="${HBASE_CONF_DIR}" 24 | 25 | # add classes first, triggers log4j.properties priority 26 | if [ -d "${bin}/../target/classes" ]; then 27 | CLASSPATH=${CLASSPATH}:${bin}/../target/classes 28 | fi 29 | 30 | # create and cache Maven classpath 31 | cpfile="${bin}/../target/cached_classpath.txt" 32 | if [ ! -f "${cpfile}" ]; then 33 | ${MVN} -f "${bin}/../pom.xml" dependency:build-classpath -Dmdep.outputFile="${cpfile}" &> /dev/null 34 | fi 35 | CLASSPATH=${CLASSPATH}:`cat "${cpfile}"` 36 | 37 | JAVA=$JAVA_HOME/bin/java 38 | JAVA_HEAP_MAX=-Xmx512m 39 | 40 | "$JAVA" $JAVA_HEAP_MAX -classpath "$CLASSPATH" "$@" -------------------------------------------------------------------------------- /ch03/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | 6 | com.larsgeorge 7 | hbase-book 8 | 2.0 9 | ../pom.xml 10 | 11 | 12 | com.larsgeorge 13 | hbase-book-ch03 14 | 2.0 15 | jar 16 | 17 | HBase Book Chapter 3 18 | 19 | 20 | 21 | com.larsgeorge 22 | hbase-book-common 23 | ${project.version} 24 | 25 | 26 | -------------------------------------------------------------------------------- /ch03/src/main/java/client/AppendExample.java: -------------------------------------------------------------------------------- 1 | package client; 2 | 3 | // cc AppendExample Example application appending data to a column in HBase 4 | 5 | import org.apache.hadoop.conf.Configuration; 6 | import org.apache.hadoop.hbase.HBaseConfiguration; 7 | import org.apache.hadoop.hbase.TableName; 8 | import org.apache.hadoop.hbase.client.Append; 9 | import org.apache.hadoop.hbase.client.Connection; 10 | import org.apache.hadoop.hbase.client.ConnectionFactory; 11 | import org.apache.hadoop.hbase.client.Table; 12 | import org.apache.hadoop.hbase.util.Bytes; 13 | import util.HBaseHelper; 14 | 15 | import java.io.IOException; 16 | 17 | public class AppendExample { 18 | 19 | public static void main(String[] args) throws IOException { 20 | Configuration conf = HBaseConfiguration.create(); 21 | 22 | HBaseHelper helper = HBaseHelper.getHelper(conf); 23 | helper.dropTable("testtable"); 24 | helper.createTable("testtable", 100, "colfam1", "colfam2"); 25 | helper.put("testtable", 26 | new String[] { "row1" }, 27 | new String[] { "colfam1" }, 28 | new String[] { "qual1" }, 29 | new long[] { 1 }, 30 | new String[] { "oldvalue" }); 31 | System.out.println("Before append call..."); 32 | helper.dump("testtable", new String[]{ "row1" }, null, null); 33 | 34 | Connection connection = ConnectionFactory.createConnection(conf); 35 | Table table = connection.getTable(TableName.valueOf("testtable")); 36 | 37 | // vv AppendExample 38 | Append append = new Append(Bytes.toBytes("row1")); 39 | append.add(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1"), 40 | Bytes.toBytes("newvalue")); 41 | append.add(Bytes.toBytes("colfam1"), Bytes.toBytes("qual2"), 42 | Bytes.toBytes("anothervalue")); 43 | 44 | table.append(append); 45 | // ^^ AppendExample 46 | System.out.println("After append call..."); 47 | helper.dump("testtable", new String[]{"row1"}, null, null); 48 | table.close(); 49 | connection.close(); 50 | helper.close(); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /ch03/src/main/java/client/CellComparatorExample.java: -------------------------------------------------------------------------------- 1 | package client; 2 | 3 | // cc CellComparatorExample Shows how to use the cell scanner 4 | import java.util.ArrayList; 5 | import java.util.Collections; 6 | 7 | import org.apache.hadoop.hbase.Cell; 8 | import org.apache.hadoop.hbase.CellComparator; 9 | import org.apache.hadoop.hbase.CellScanner; 10 | import org.apache.hadoop.hbase.client.Put; 11 | import org.apache.hadoop.hbase.util.Bytes; 12 | 13 | public class CellComparatorExample { 14 | 15 | public static void main(String[] args) throws Exception { 16 | // vv CellComparatorExample 17 | Put put1 = new Put(Bytes.toBytes("row-1")); 18 | put1.addColumn(Bytes.toBytes("fam-1"), Bytes.toBytes("qual-1"), 19 | Bytes.toBytes("val-1")); 20 | Put put2 = new Put(Bytes.toBytes("row-2")); 21 | put2.addColumn(Bytes.toBytes("fam-1"), Bytes.toBytes("qual-2"), 22 | Bytes.toBytes("val-2")); 23 | Put put3 = new Put(Bytes.toBytes("row-3")); 24 | put3.addColumn(Bytes.toBytes("fam-2"), Bytes.toBytes("qual-3"), 25 | Bytes.toBytes("val-3")); 26 | Put put4 = new Put(Bytes.toBytes("row-1")); 27 | put4.addColumn(Bytes.toBytes("fam-2"), Bytes.toBytes("qual-2"), 28 | Bytes.toBytes("val-2")); 29 | 30 | CellComparator comparator = new CellComparator.RowComparator(); 31 | ArrayList cells = new ArrayList<>(); 32 | 33 | Put[] puts = { put1, put2, put3, put4 }; 34 | 35 | for (Put put : puts) { 36 | CellScanner scanner = put.cellScanner(); 37 | while (scanner.advance()) { 38 | Cell cell = scanner.current(); 39 | cells.add(cell); 40 | } 41 | } 42 | 43 | System.out.println("Shuffling..."); 44 | Collections.shuffle(cells); 45 | for (Cell cell : cells) { 46 | System.out.println("Cell: " + cell); 47 | } 48 | 49 | System.out.println("Sorting..."); 50 | Collections.sort(cells, comparator); 51 | for (Cell cell : cells) { 52 | System.out.println("Cell: " + cell); 53 | } 54 | // ^^ CellComparatorExample 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /ch03/src/main/java/client/CellScannerExample.java: -------------------------------------------------------------------------------- 1 | package client; 2 | 3 | // cc CellScannerExample Shows how to use the cell scanner 4 | import org.apache.hadoop.hbase.Cell; 5 | import org.apache.hadoop.hbase.CellScanner; 6 | import org.apache.hadoop.hbase.client.Put; 7 | import org.apache.hadoop.hbase.util.Bytes; 8 | 9 | public class CellScannerExample { 10 | 11 | public static void main(String[] args) throws Exception { 12 | // vv CellScannerExample 13 | Put put = new Put(Bytes.toBytes("testrow")); 14 | put.addColumn(Bytes.toBytes("fam-1"), Bytes.toBytes("qual-1"), 15 | Bytes.toBytes("val-1")); 16 | put.addColumn(Bytes.toBytes("fam-1"), Bytes.toBytes("qual-2"), 17 | Bytes.toBytes("val-2")); 18 | put.addColumn(Bytes.toBytes("fam-2"), Bytes.toBytes("qual-3"), 19 | Bytes.toBytes("val-3")); 20 | 21 | CellScanner scanner = put.cellScanner(); 22 | while (scanner.advance()) { 23 | Cell cell = scanner.current(); 24 | System.out.println("Cell: " + cell); 25 | } 26 | // ^^ CellScannerExample 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /ch03/src/main/java/client/FingerprintExample.java: -------------------------------------------------------------------------------- 1 | package client; 2 | 3 | // cc FingerprintExample Shows what the fingerprint and ID of a data class comprises 4 | import java.net.InetAddress; 5 | 6 | import org.apache.hadoop.hbase.client.Put; 7 | import org.apache.hadoop.hbase.util.Bytes; 8 | 9 | public class FingerprintExample { 10 | 11 | public static void main(String[] args) throws Exception { 12 | // vv FingerprintExample 13 | Put put = new Put(Bytes.toBytes("testrow")); 14 | put.addColumn(Bytes.toBytes("fam-1"), Bytes.toBytes("qual-1"), 15 | Bytes.toBytes("val-1")); 16 | put.addColumn(Bytes.toBytes("fam-1"), Bytes.toBytes("qual-2"), 17 | Bytes.toBytes("val-2")); 18 | put.addColumn(Bytes.toBytes("fam-2"), Bytes.toBytes("qual-3"), 19 | Bytes.toBytes("val-3")); 20 | 21 | String id = String.format("Hostname: %s, App: %s", 22 | InetAddress.getLocalHost().getHostName(), 23 | System.getProperty("sun.java.command")); 24 | put.setId(id); 25 | 26 | System.out.println("Put.size: " + put.size()); 27 | System.out.println("Put.id: " + put.getId()); 28 | System.out.println("Put.fingerprint: " + put.getFingerprint()); 29 | System.out.println("Put.toMap: " + put.toMap()); 30 | System.out.println("Put.toJSON: " + put.toJSON()); 31 | System.out.println("Put.toString: " + put.toString()); 32 | // ^^ FingerprintExample 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /ch03/src/main/java/client/GetCloneExample.java: -------------------------------------------------------------------------------- 1 | package client; 2 | 3 | // cc GetCloneExample Example application retrieving data from HBase 4 | import java.io.IOException; 5 | 6 | import org.apache.hadoop.conf.Configuration; 7 | import org.apache.hadoop.hbase.HBaseConfiguration; 8 | import org.apache.hadoop.hbase.TableName; 9 | import org.apache.hadoop.hbase.client.Connection; 10 | import org.apache.hadoop.hbase.client.ConnectionFactory; 11 | import org.apache.hadoop.hbase.client.Get; 12 | import org.apache.hadoop.hbase.client.Result; 13 | import org.apache.hadoop.hbase.client.Table; 14 | import org.apache.hadoop.hbase.util.Bytes; 15 | 16 | import util.HBaseHelper; 17 | 18 | public class GetCloneExample { 19 | 20 | public static void main(String[] args) throws IOException { 21 | Configuration conf = HBaseConfiguration.create(); 22 | 23 | HBaseHelper helper = HBaseHelper.getHelper(conf); 24 | if (!helper.existsTable("testtable")) { 25 | helper.createTable("testtable", "colfam1"); 26 | } 27 | Connection connection = ConnectionFactory.createConnection(conf); 28 | Table table = connection.getTable(TableName.valueOf("testtable")); 29 | 30 | // vv GetCloneExample 31 | Get get1 = new Get(Bytes.toBytes("row1")); 32 | get1.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1")); 33 | 34 | Get get2 = new Get(get1); 35 | Result result = table.get(get2); 36 | 37 | System.out.println("Result : " + result); 38 | // ^^ GetCloneExample 39 | table.close(); 40 | connection.close(); 41 | helper.close(); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /ch03/src/main/java/client/GetExample.java: -------------------------------------------------------------------------------- 1 | package client; 2 | 3 | // cc GetExample Example application retrieving data from HBase 4 | import java.io.IOException; 5 | 6 | import org.apache.hadoop.conf.Configuration; 7 | import org.apache.hadoop.hbase.HBaseConfiguration; 8 | import org.apache.hadoop.hbase.TableName; 9 | import org.apache.hadoop.hbase.client.Connection; 10 | import org.apache.hadoop.hbase.client.ConnectionFactory; 11 | import org.apache.hadoop.hbase.client.Get; 12 | import org.apache.hadoop.hbase.client.Result; 13 | import org.apache.hadoop.hbase.client.Table; 14 | import org.apache.hadoop.hbase.util.Bytes; 15 | 16 | import util.HBaseHelper; 17 | 18 | public class GetExample { 19 | 20 | public static void main(String[] args) throws IOException { 21 | // vv GetExample 22 | Configuration conf = HBaseConfiguration.create(); // co GetExample-1-CreateConf Create the configuration. 23 | 24 | // ^^ GetExample 25 | HBaseHelper helper = HBaseHelper.getHelper(conf); 26 | if (!helper.existsTable("testtable")) { 27 | helper.createTable("testtable", "colfam1"); 28 | } 29 | // vv GetExample 30 | Connection connection = ConnectionFactory.createConnection(conf); 31 | Table table = connection.getTable(TableName.valueOf("testtable")); // co GetExample-2-NewTable Instantiate a new table reference. 32 | 33 | Get get = new Get(Bytes.toBytes("row1")); // co GetExample-3-NewGet Create get with specific row. 34 | 35 | get.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1")); // co GetExample-4-AddCol Add a column to the get. 36 | 37 | Result result = table.get(get); // co GetExample-5-DoGet Retrieve row with selected columns from HBase. 38 | 39 | byte[] val = result.getValue(Bytes.toBytes("colfam1"), 40 | Bytes.toBytes("qual1")); // co GetExample-6-GetValue Get a specific value for the given column. 41 | 42 | System.out.println("Value: " + Bytes.toString(val)); // co GetExample-7-Print Print out the value while converting it back. 43 | 44 | table.close(); // co GetExample-8-Close Close the table and connection instances to free resources. 45 | connection.close(); 46 | // ^^ GetExample 47 | helper.close(); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /ch03/src/main/java/client/GetFluentExample.java: -------------------------------------------------------------------------------- 1 | package client; 2 | 3 | // cc GetFluentExample Creates a get request using its fluent interface 4 | import java.io.IOException; 5 | 6 | import org.apache.hadoop.conf.Configuration; 7 | import org.apache.hadoop.hbase.HBaseConfiguration; 8 | import org.apache.hadoop.hbase.TableName; 9 | import org.apache.hadoop.hbase.client.Connection; 10 | import org.apache.hadoop.hbase.client.ConnectionFactory; 11 | import org.apache.hadoop.hbase.client.Get; 12 | import org.apache.hadoop.hbase.client.Result; 13 | import org.apache.hadoop.hbase.client.Table; 14 | import org.apache.hadoop.hbase.util.Bytes; 15 | 16 | import util.HBaseHelper; 17 | 18 | public class GetFluentExample { 19 | 20 | public static void main(String[] args) throws IOException { 21 | Configuration conf = HBaseConfiguration.create(); 22 | 23 | HBaseHelper helper = HBaseHelper.getHelper(conf); 24 | helper.dropTable("testtable"); 25 | helper.createTable("testtable", 5, "colfam1", "colfam2"); 26 | helper.put("testtable", 27 | new String[] { "row1" }, 28 | new String[] { "colfam1", "colfam2" }, 29 | new String[] { "qual1", "qual1", "qual2", "qual2" }, 30 | new long[] { 1, 2, 3, 4 }, 31 | new String[] { "val1", "val1", "val2", "val2" }); 32 | System.out.println("Before get call..."); 33 | helper.dump("testtable", new String[]{ "row1" }, null, null); 34 | Connection connection = ConnectionFactory.createConnection(conf); 35 | Table table = connection.getTable(TableName.valueOf("testtable")); 36 | 37 | // vv GetFluentExample 38 | Get get = new Get(Bytes.toBytes("row1")) // co GetFluentExample-1-Create Create a new get using the fluent interface. 39 | .setId("GetFluentExample") 40 | .setMaxVersions() 41 | .setTimeStamp(1) 42 | .addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1")) 43 | .addFamily(Bytes.toBytes("colfam2")); 44 | 45 | Result result = table.get(get); 46 | System.out.println("Result: " + result); 47 | // ^^ GetFluentExample 48 | table.close(); 49 | connection.close(); 50 | helper.close(); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /ch03/src/main/java/client/GetListErrorExample.java: -------------------------------------------------------------------------------- 1 | package client; 2 | 3 | // cc GetListErrorExample Example trying to read an erroneous column family 4 | import java.io.IOException; 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | 8 | import org.apache.hadoop.conf.Configuration; 9 | import org.apache.hadoop.hbase.HBaseConfiguration; 10 | import org.apache.hadoop.hbase.TableName; 11 | import org.apache.hadoop.hbase.client.Connection; 12 | import org.apache.hadoop.hbase.client.ConnectionFactory; 13 | import org.apache.hadoop.hbase.client.Get; 14 | import org.apache.hadoop.hbase.client.Result; 15 | import org.apache.hadoop.hbase.client.Table; 16 | import org.apache.hadoop.hbase.util.Bytes; 17 | 18 | import util.HBaseHelper; 19 | 20 | public class GetListErrorExample { 21 | 22 | public static void main(String[] args) throws IOException { 23 | Configuration conf = HBaseConfiguration.create(); 24 | 25 | HBaseHelper helper = HBaseHelper.getHelper(conf); 26 | if (!helper.existsTable("testtable")) { 27 | helper.createTable("testtable", "colfam1"); 28 | } 29 | 30 | Connection connection = ConnectionFactory.createConnection(conf); 31 | Table table = connection.getTable(TableName.valueOf("testtable")); 32 | 33 | byte[] cf1 = Bytes.toBytes("colfam1"); 34 | byte[] qf1 = Bytes.toBytes("qual1"); 35 | byte[] qf2 = Bytes.toBytes("qual2"); 36 | byte[] row1 = Bytes.toBytes("row1"); 37 | byte[] row2 = Bytes.toBytes("row2"); 38 | 39 | // vv GetListErrorExample 40 | List gets = new ArrayList(); 41 | 42 | Get get1 = new Get(row1); 43 | get1.addColumn(cf1, qf1); 44 | gets.add(get1); 45 | 46 | Get get2 = new Get(row2); 47 | get2.addColumn(cf1, qf1); // co GetListErrorExample-1-AddGets Add the Get instances to the list. 48 | gets.add(get2); 49 | 50 | Get get3 = new Get(row2); 51 | get3.addColumn(cf1, qf2); 52 | gets.add(get3); 53 | 54 | Get get4 = new Get(row2); 55 | /*[*/get4.addColumn(Bytes.toBytes("BOGUS"),/*]*/ qf2); 56 | gets.add(get4); // co GetListErrorExample-2-AddBogus Add the bogus column family get. 57 | 58 | Result[] results = table.get(gets); // co GetListErrorExample-3-Error An exception is thrown and the process is aborted. 59 | 60 | System.out.println("Result count: " + results.length); // co GetListErrorExample-4-SOUT This line will never reached! 61 | // ^^ GetListErrorExample 62 | table.close(); 63 | connection.close(); 64 | helper.close(); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /ch03/src/main/java/client/GetTryWithResourcesExample.java: -------------------------------------------------------------------------------- 1 | package client; 2 | 3 | // cc GetTryWithResourcesExample Example application retrieving data from HBase using a Java 7 construct 4 | import org.apache.hadoop.conf.Configuration; 5 | import org.apache.hadoop.hbase.HBaseConfiguration; 6 | import org.apache.hadoop.hbase.TableName; 7 | import org.apache.hadoop.hbase.client.Connection; 8 | import org.apache.hadoop.hbase.client.ConnectionFactory; 9 | import org.apache.hadoop.hbase.client.Get; 10 | import org.apache.hadoop.hbase.client.Result; 11 | import org.apache.hadoop.hbase.client.Table; 12 | import org.apache.hadoop.hbase.util.Bytes; 13 | import util.HBaseHelper; 14 | 15 | import java.io.IOException; 16 | 17 | public class GetTryWithResourcesExample { 18 | 19 | public static void main(String[] args) throws IOException { 20 | // vv GetTryWithResourcesExample 21 | Configuration conf = HBaseConfiguration.create(); // co GetTryWithResourcesExample-1-CreateConf Create the configuration. 22 | 23 | // ^^ GetTryWithResourcesExample 24 | HBaseHelper helper = HBaseHelper.getHelper(conf); 25 | if (!helper.existsTable("testtable")) { 26 | helper.createTable("testtable", "colfam1"); 27 | } 28 | // vv GetTryWithResourcesExample 29 | try ( 30 | Connection connection = ConnectionFactory.createConnection(conf); 31 | Table table = connection.getTable(TableName.valueOf("testtable")); // co GetTryWithResourcesExample-2-NewTable Instantiate a new table reference in "try" block. 32 | ) { 33 | Get get = new Get(Bytes.toBytes("row1")); 34 | get.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1")); 35 | Result result = table.get(get); 36 | byte[] val = result.getValue(Bytes.toBytes("colfam1"), 37 | Bytes.toBytes("qual1")); 38 | System.out.println("Value: " + Bytes.toString(val)); 39 | } // co GetTryWithResourcesExample-3-Close No explicit close needed, Java will handle AutoClosable's. 40 | // ^^ GetTryWithResourcesExample 41 | helper.close(); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /ch03/src/main/java/client/MutateRowExample.java: -------------------------------------------------------------------------------- 1 | package client; 2 | 3 | // cc MutateRowExample Modifies a row with multiple operations 4 | import java.io.IOException; 5 | 6 | import org.apache.hadoop.conf.Configuration; 7 | import org.apache.hadoop.hbase.HBaseConfiguration; 8 | import org.apache.hadoop.hbase.TableName; 9 | import org.apache.hadoop.hbase.client.Connection; 10 | import org.apache.hadoop.hbase.client.ConnectionFactory; 11 | import org.apache.hadoop.hbase.client.Delete; 12 | import org.apache.hadoop.hbase.client.Put; 13 | import org.apache.hadoop.hbase.client.RowMutations; 14 | import org.apache.hadoop.hbase.client.Table; 15 | import org.apache.hadoop.hbase.util.Bytes; 16 | 17 | import util.HBaseHelper; 18 | 19 | public class MutateRowExample { 20 | 21 | public static void main(String[] args) throws IOException { 22 | Configuration conf = HBaseConfiguration.create(); 23 | 24 | HBaseHelper helper = HBaseHelper.getHelper(conf); 25 | helper.dropTable("testtable"); 26 | helper.createTable("testtable", 3, "colfam1"); 27 | helper.put("testtable", 28 | new String[] { "row1" }, 29 | new String[] { "colfam1" }, 30 | new String[] { "qual1", "qual2", "qual3" }, 31 | new long[] { 1, 2, 3 }, 32 | new String[] { "val1", "val2", "val3" }); 33 | System.out.println("Before delete call..."); 34 | helper.dump("testtable", new String[]{"row1"}, null, null); 35 | 36 | Connection connection = ConnectionFactory.createConnection(conf); 37 | Table table = connection.getTable(TableName.valueOf("testtable")); 38 | 39 | // vv MutateRowExample 40 | Put put = new Put(Bytes.toBytes("row1")); 41 | put.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1"), 42 | 4, Bytes.toBytes("val99")); 43 | put.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual4"), 44 | 4, Bytes.toBytes("val100")); 45 | 46 | Delete delete = new Delete(Bytes.toBytes("row1")); 47 | delete.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual2")); 48 | 49 | RowMutations mutations = new RowMutations(Bytes.toBytes("row1")); 50 | mutations.add(put); 51 | mutations.add(delete); 52 | 53 | table.mutateRow(mutations); 54 | // ^^ MutateRowExample 55 | table.close(); 56 | connection.close(); 57 | System.out.println("After mutate call..."); 58 | helper.dump("testtable", new String[] { "row1" }, null, null); 59 | helper.close(); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /ch03/src/main/java/client/PutExample.java: -------------------------------------------------------------------------------- 1 | package client; 2 | 3 | // cc PutExample Example application inserting data into HBase 4 | // vv PutExample 5 | import org.apache.hadoop.conf.Configuration; 6 | import org.apache.hadoop.hbase.HBaseConfiguration; 7 | import org.apache.hadoop.hbase.TableName; 8 | import org.apache.hadoop.hbase.client.Connection; 9 | import org.apache.hadoop.hbase.client.ConnectionFactory; 10 | import org.apache.hadoop.hbase.client.Put; 11 | import org.apache.hadoop.hbase.client.Table; 12 | import org.apache.hadoop.hbase.util.Bytes; 13 | // ^^ PutExample 14 | import util.HBaseHelper; 15 | // vv PutExample 16 | 17 | import java.io.IOException; 18 | 19 | public class PutExample { 20 | 21 | public static void main(String[] args) throws IOException { 22 | Configuration conf = HBaseConfiguration.create(); // co PutExample-1-CreateConf Create the required configuration. 23 | 24 | // ^^ PutExample 25 | HBaseHelper helper = HBaseHelper.getHelper(conf); 26 | helper.dropTable("testtable"); 27 | helper.createTable("testtable", "colfam1"); 28 | // vv PutExample 29 | Connection connection = ConnectionFactory.createConnection(conf); 30 | Table table = connection.getTable(TableName.valueOf("testtable")); // co PutExample-2-NewTable Instantiate a new client. 31 | 32 | Put put = new Put(Bytes.toBytes("row1")); // co PutExample-3-NewPut Create put with specific row. 33 | 34 | put.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1"), 35 | Bytes.toBytes("val1")); // co PutExample-4-AddCol1 Add a column, whose name is "colfam1:qual1", to the put. 36 | put.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual2"), 37 | Bytes.toBytes("val2")); // co PutExample-4-AddCol2 Add another column, whose name is "colfam1:qual2", to the put. 38 | 39 | table.put(put); // co PutExample-5-DoPut Store row with column into the HBase table. 40 | table.close(); // co PutExample-6-DoPut Close table and connection instances to free resources. 41 | connection.close(); 42 | // ^^ PutExample 43 | helper.close(); 44 | // vv PutExample 45 | } 46 | } 47 | // ^^ PutExample 48 | -------------------------------------------------------------------------------- /ch03/src/main/java/client/PutIdenticalExample.java: -------------------------------------------------------------------------------- 1 | package client; 2 | 3 | // cc PutIdenticalExample Example adding an identical column twice 4 | import org.apache.hadoop.conf.Configuration; 5 | import org.apache.hadoop.hbase.HBaseConfiguration; 6 | import org.apache.hadoop.hbase.TableName; 7 | import org.apache.hadoop.hbase.client.Connection; 8 | import org.apache.hadoop.hbase.client.ConnectionFactory; 9 | import org.apache.hadoop.hbase.client.Get; 10 | import org.apache.hadoop.hbase.client.Put; 11 | import org.apache.hadoop.hbase.client.Result; 12 | import org.apache.hadoop.hbase.client.Table; 13 | import org.apache.hadoop.hbase.util.Bytes; 14 | import util.HBaseHelper; 15 | 16 | import java.io.IOException; 17 | 18 | public class PutIdenticalExample { 19 | 20 | public static void main(String[] args) throws IOException { 21 | Configuration conf = HBaseConfiguration.create(); 22 | 23 | HBaseHelper helper = HBaseHelper.getHelper(conf); 24 | helper.dropTable("testtable"); 25 | helper.createTable("testtable", "colfam1"); 26 | 27 | Connection connection = ConnectionFactory.createConnection(conf); 28 | Table table = connection.getTable(TableName.valueOf("testtable")); 29 | 30 | // vv PutIdenticalExample 31 | Put put = new Put(Bytes.toBytes("row1")); 32 | put.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1"), 33 | Bytes.toBytes("val2")); 34 | put.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1"), 35 | Bytes.toBytes("val1")); // co PutIdenticalExample-1-Add Add the same column with a different value. The last value is going to be used. 36 | table.put(put); 37 | 38 | Get get = new Get(Bytes.toBytes("row1")); 39 | Result result = table.get(get); 40 | System.out.println("Result: " + result + ", Value: " + Bytes.toString( 41 | result.getValue(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1")))); // co PutIdenticalExample-2-Get Perform a get to verify that "val1" was actually stored. 42 | // ^^ PutIdenticalExample 43 | table.close(); 44 | connection.close(); 45 | helper.close(); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /ch03/src/main/java/client/PutListErrorExample1.java: -------------------------------------------------------------------------------- 1 | package client; 2 | 3 | // cc PutListErrorExample1 Example inserting a faulty column family into HBase 4 | import java.io.IOException; 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | 8 | import org.apache.hadoop.conf.Configuration; 9 | import org.apache.hadoop.hbase.HBaseConfiguration; 10 | import org.apache.hadoop.hbase.TableName; 11 | import org.apache.hadoop.hbase.client.Connection; 12 | import org.apache.hadoop.hbase.client.ConnectionFactory; 13 | import org.apache.hadoop.hbase.client.Put; 14 | import org.apache.hadoop.hbase.client.Table; 15 | import org.apache.hadoop.hbase.util.Bytes; 16 | 17 | import util.HBaseHelper; 18 | 19 | public class PutListErrorExample1 { 20 | 21 | public static void main(String[] args) throws IOException { 22 | Configuration conf = HBaseConfiguration.create(); 23 | 24 | HBaseHelper helper = HBaseHelper.getHelper(conf); 25 | helper.dropTable("testtable"); 26 | helper.createTable("testtable", "colfam1"); 27 | 28 | Connection connection = ConnectionFactory.createConnection(conf); 29 | Table table = connection.getTable(TableName.valueOf("testtable")); 30 | 31 | List puts = new ArrayList(); 32 | 33 | // vv PutListErrorExample1 34 | Put put1 = new Put(Bytes.toBytes("row1")); 35 | put1.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1"), 36 | Bytes.toBytes("val1")); 37 | puts.add(put1); 38 | Put put2 = new Put(Bytes.toBytes("row2")); 39 | /*[*/put2.addColumn(Bytes.toBytes("BOGUS"),/*]*/ Bytes.toBytes("qual1"), 40 | Bytes.toBytes("val2")); // co PutListErrorExample1-1-AddErrorPut Add put with non existent family to list. 41 | puts.add(put2); 42 | Put put3 = new Put(Bytes.toBytes("row2")); 43 | put3.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual2"), 44 | Bytes.toBytes("val3")); 45 | puts.add(put3); 46 | 47 | table.put(puts); // co PutListErrorExample1-2-DoPut Store multiple rows with columns into HBase. 48 | // ^^ PutListErrorExample1 49 | table.close(); 50 | connection.close(); 51 | helper.close(); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /ch03/src/main/java/client/PutListErrorExample2.java: -------------------------------------------------------------------------------- 1 | package client; 2 | 3 | // cc PutListErrorExample2 Example inserting an empty Put instance into HBase 4 | import java.io.IOException; 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | 8 | import org.apache.hadoop.conf.Configuration; 9 | import org.apache.hadoop.hbase.HBaseConfiguration; 10 | import org.apache.hadoop.hbase.TableName; 11 | import org.apache.hadoop.hbase.client.Connection; 12 | import org.apache.hadoop.hbase.client.ConnectionFactory; 13 | import org.apache.hadoop.hbase.client.Put; 14 | import org.apache.hadoop.hbase.client.Table; 15 | import org.apache.hadoop.hbase.util.Bytes; 16 | 17 | import util.HBaseHelper; 18 | 19 | public class PutListErrorExample2 { 20 | 21 | public static void main(String[] args) throws IOException { 22 | Configuration conf = HBaseConfiguration.create(); 23 | 24 | HBaseHelper helper = HBaseHelper.getHelper(conf); 25 | helper.dropTable("testtable"); 26 | helper.createTable("testtable", "colfam1"); 27 | 28 | Connection connection = ConnectionFactory.createConnection(conf); 29 | Table table = connection.getTable(TableName.valueOf("testtable")); 30 | 31 | List puts = new ArrayList(); 32 | 33 | // vv PutListErrorExample2 34 | Put put1 = new Put(Bytes.toBytes("row1")); 35 | put1.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1"), 36 | Bytes.toBytes("val1")); 37 | puts.add(put1); 38 | Put put2 = new Put(Bytes.toBytes("row2")); 39 | put2.addColumn(Bytes.toBytes("BOGUS"), Bytes.toBytes("qual1"), 40 | Bytes.toBytes("val2")); 41 | puts.add(put2); 42 | Put put3 = new Put(Bytes.toBytes("row2")); 43 | put3.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual2"), 44 | Bytes.toBytes("val3")); 45 | puts.add(put3); 46 | /*[*/Put put4 = new Put(Bytes.toBytes("row2")); 47 | puts.add(put4);/*]*/ // co PutListErrorExample2-1-AddErrorPut Add put with no content at all to list. 48 | 49 | /*[*/try {/*]*/ 50 | table.put(puts); 51 | /*[*/} catch (Exception e) { 52 | System.err.println("Error: " + e); 53 | // table.flushCommits(); 54 | // todo: FIX! 55 | /*]*/ // co PutListErrorExample2-2-Catch Catch local exception and commit queued updates. 56 | /*[*/}/*]*/ 57 | // ^^ PutListErrorExample2 58 | table.close(); 59 | connection.close(); 60 | helper.close(); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /ch03/src/main/java/client/PutListExample.java: -------------------------------------------------------------------------------- 1 | package client; 2 | 3 | // cc PutListExample Example inserting data into HBase using a list 4 | import java.io.IOException; 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | 8 | import org.apache.hadoop.conf.Configuration; 9 | import org.apache.hadoop.hbase.HBaseConfiguration; 10 | import org.apache.hadoop.hbase.TableName; 11 | import org.apache.hadoop.hbase.client.Connection; 12 | import org.apache.hadoop.hbase.client.ConnectionFactory; 13 | import org.apache.hadoop.hbase.client.Put; 14 | import org.apache.hadoop.hbase.client.Table; 15 | import org.apache.hadoop.hbase.util.Bytes; 16 | 17 | import util.HBaseHelper; 18 | 19 | public class PutListExample { 20 | 21 | public static void main(String[] args) throws IOException { 22 | Configuration conf = HBaseConfiguration.create(); 23 | 24 | HBaseHelper helper = HBaseHelper.getHelper(conf); 25 | helper.dropTable("testtable"); 26 | helper.createTable("testtable", "colfam1"); 27 | 28 | Connection connection = ConnectionFactory.createConnection(conf); 29 | Table table = connection.getTable(TableName.valueOf("testtable")); 30 | 31 | // vv PutListExample 32 | List puts = new ArrayList(); // co PutListExample-1-CreateList Create a list that holds the Put instances. 33 | 34 | Put put1 = new Put(Bytes.toBytes("row1")); 35 | put1.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1"), 36 | Bytes.toBytes("val1")); 37 | puts.add(put1); // co PutListExample-2-AddPut1 Add put to list. 38 | 39 | Put put2 = new Put(Bytes.toBytes("row2")); 40 | put2.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1"), 41 | Bytes.toBytes("val2")); 42 | puts.add(put2); // co PutListExample-3-AddPut2 Add another put to list. 43 | 44 | Put put3 = new Put(Bytes.toBytes("row2")); 45 | put3.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual2"), 46 | Bytes.toBytes("val3")); 47 | puts.add(put3); // co PutListExample-4-AddPut3 Add third put to list. 48 | 49 | table.put(puts); // co PutListExample-5-DoPut Store multiple rows with columns into HBase. 50 | // ^^ PutListExample 51 | table.close(); 52 | connection.close(); 53 | helper.close(); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /ch03/src/main/java/client/ResultExample.java: -------------------------------------------------------------------------------- 1 | package client; 2 | 3 | // cc ResultExample Retrieve results from server and dump content 4 | import java.io.IOException; 5 | 6 | import org.apache.hadoop.conf.Configuration; 7 | import org.apache.hadoop.hbase.HBaseConfiguration; 8 | import org.apache.hadoop.hbase.TableName; 9 | import org.apache.hadoop.hbase.client.Connection; 10 | import org.apache.hadoop.hbase.client.ConnectionFactory; 11 | import org.apache.hadoop.hbase.client.Get; 12 | import org.apache.hadoop.hbase.client.Put; 13 | import org.apache.hadoop.hbase.client.Result; 14 | import org.apache.hadoop.hbase.client.Table; 15 | import org.apache.hadoop.hbase.util.Bytes; 16 | 17 | import util.HBaseHelper; 18 | 19 | public class ResultExample { 20 | 21 | public static void main(String[] args) throws IOException { 22 | Configuration conf = HBaseConfiguration.create(); 23 | 24 | HBaseHelper helper = HBaseHelper.getHelper(conf); 25 | helper.dropTable("testtable"); 26 | helper.createTable("testtable", "colfam1"); 27 | Connection connection = ConnectionFactory.createConnection(conf); 28 | Table table = connection.getTable(TableName.valueOf("testtable")); 29 | 30 | Put put = new Put(Bytes.toBytes("row1")); 31 | put.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1"), 32 | Bytes.toBytes("val1")); 33 | put.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual2"), 34 | Bytes.toBytes("val2")); 35 | table.put(put); 36 | 37 | // vv ResultExample 38 | Get get = new Get(Bytes.toBytes("row1")); 39 | Result result1 = table.get(get); 40 | System.out.println(result1); 41 | 42 | Result result2 = Result.EMPTY_RESULT; 43 | System.out.println(result2); 44 | 45 | result2.copyFrom(result1); 46 | System.out.println(result2); 47 | 48 | // ^^ ResultExample 49 | table.close(); 50 | connection.close(); 51 | helper.close(); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /ch03/src/main/java/client/RowKeyExample.java: -------------------------------------------------------------------------------- 1 | package client; 2 | 3 | // cc RowKeyExample Example row key usage from existing array 4 | import java.nio.charset.Charset; 5 | import java.util.Arrays; 6 | 7 | import org.apache.hadoop.hbase.client.Put; 8 | import org.apache.hadoop.hbase.util.Bytes; 9 | 10 | public class RowKeyExample { 11 | 12 | public static void main(String[] args) { 13 | // vv RowKeyExample 14 | byte[] data = new byte[100]; 15 | Arrays.fill(data, (byte) '@'); 16 | String username = "johndoe"; 17 | byte[] username_bytes = username.getBytes(Charset.forName("UTF8")); 18 | 19 | System.arraycopy(username_bytes, 0, data, 45, username_bytes.length); 20 | System.out.println("data length: " + data.length + 21 | ", data: " + Bytes.toString(data)); 22 | 23 | Put put = new Put(data, 45, username_bytes.length); 24 | System.out.println("Put: " + put); 25 | // ^^ RowKeyExample 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /ch03/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO, stdout 2 | 3 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 4 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 5 | 6 | # Pattern to output the caller's file name and line number. 7 | log4j.appender.stdout.layout.ConversionPattern=%5p [%t] (%F:%L) - %m%n 8 | 9 | log4j.logger.org.apache=WARN -------------------------------------------------------------------------------- /ch04/bin/run.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | # 3 | # run.sh: helps executing the examples by setting up the Java CLASSPATH. 4 | # 5 | 6 | # get the current directory 7 | bin=`dirname "$0"` 8 | bin=`cd "$bin">/dev/null; pwd` 9 | 10 | # check command line args 11 | if [[ $# == 0 ]]; then 12 | echo "usage: $(basename $0) " 13 | exit 1; 14 | fi 15 | 16 | # set up Maven environment 17 | MVN="mvn" 18 | if [ "$MAVEN_HOME" != "" ]; then 19 | MVN=${MAVEN_HOME}/bin/mvn 20 | fi 21 | 22 | # classpath initially contains $HBASE_CONF_DIR 23 | CLASSPATH="${HBASE_CONF_DIR}" 24 | 25 | # add classes first, triggers log4j.properties priority 26 | if [ -d "${bin}/../target/classes" ]; then 27 | CLASSPATH=${CLASSPATH}:${bin}/../target/classes 28 | fi 29 | 30 | # create and cache Maven classpath 31 | cpfile="${bin}/../target/cached_classpath.txt" 32 | if [ ! -f "${cpfile}" ]; then 33 | ${MVN} -f "${bin}/../pom.xml" dependency:build-classpath -Dmdep.outputFile="${cpfile}" &> /dev/null 34 | fi 35 | CLASSPATH=${CLASSPATH}:`cat "${cpfile}"` 36 | 37 | JAVA=$JAVA_HOME/bin/java 38 | JAVA_HEAP_MAX=-Xmx512m 39 | 40 | "$JAVA" $JAVA_HEAP_MAX -classpath "$CLASSPATH" "$@" -------------------------------------------------------------------------------- /ch04/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | 6 | com.larsgeorge 7 | hbase-book 8 | 2.0 9 | ../pom.xml 10 | 11 | 12 | com.larsgeorge 13 | hbase-book-ch04 14 | 2.0 15 | jar 16 | 17 | HBase Book Chapter 4 18 | 19 | 20 | 21 | com.larsgeorge 22 | hbase-book-common 23 | ${project.version} 24 | 25 | 26 | -------------------------------------------------------------------------------- /ch04/src/main/java/client/IncrementSingleExample.java: -------------------------------------------------------------------------------- 1 | package client; 2 | 3 | // cc IncrementSingleExample Example using the single counter increment methods 4 | import java.io.IOException; 5 | 6 | import org.apache.hadoop.conf.Configuration; 7 | import org.apache.hadoop.hbase.HBaseConfiguration; 8 | import org.apache.hadoop.hbase.TableName; 9 | import org.apache.hadoop.hbase.client.Connection; 10 | import org.apache.hadoop.hbase.client.ConnectionFactory; 11 | import org.apache.hadoop.hbase.client.Table; 12 | import org.apache.hadoop.hbase.util.Bytes; 13 | 14 | import util.HBaseHelper; 15 | 16 | public class IncrementSingleExample { 17 | 18 | public static void main(String[] args) throws IOException { 19 | Configuration conf = HBaseConfiguration.create(); 20 | 21 | HBaseHelper helper = HBaseHelper.getHelper(conf); 22 | helper.dropTable("testtable"); 23 | helper.createTable("testtable", "daily"); 24 | Connection connection = ConnectionFactory.createConnection(conf); 25 | Table table = connection.getTable(TableName.valueOf("testtable")); 26 | // vv IncrementSingleExample 27 | long cnt1 = table.incrementColumnValue(Bytes.toBytes("20110101"), // co IncrementSingleExample-1-Incr1 Increase counter by one. 28 | Bytes.toBytes("daily"), Bytes.toBytes("hits"), 1); 29 | long cnt2 = table.incrementColumnValue(Bytes.toBytes("20110101"), // co IncrementSingleExample-2-Incr2 Increase counter by one a second time. 30 | Bytes.toBytes("daily"), Bytes.toBytes("hits"), 1); 31 | 32 | long current = table.incrementColumnValue(Bytes.toBytes("20110101"), // co IncrementSingleExample-3-GetCurrent Get current value of the counter without increasing it. 33 | Bytes.toBytes("daily"), Bytes.toBytes("hits"), 0); 34 | 35 | long cnt3 = table.incrementColumnValue(Bytes.toBytes("20110101"), // co IncrementSingleExample-4-Decr1 Decrease counter by one. 36 | Bytes.toBytes("daily"), Bytes.toBytes("hits"), -1); 37 | // ^^ IncrementSingleExample 38 | System.out.println("cnt1: " + cnt1 + ", cnt2: " + cnt2 + 39 | ", current: " + current + ", cnt3: " + cnt3); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /ch04/src/main/java/coprocessor/LoadWithTableDescriptorExample.java: -------------------------------------------------------------------------------- 1 | package coprocessor; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.conf.Configuration; 6 | import org.apache.hadoop.hbase.Coprocessor; 7 | import org.apache.hadoop.hbase.HBaseConfiguration; 8 | import org.apache.hadoop.hbase.HColumnDescriptor; 9 | import org.apache.hadoop.hbase.HTableDescriptor; 10 | import org.apache.hadoop.hbase.TableName; 11 | import org.apache.hadoop.hbase.client.Admin; 12 | import org.apache.hadoop.hbase.client.Connection; 13 | import org.apache.hadoop.hbase.client.ConnectionFactory; 14 | 15 | import util.HBaseHelper; 16 | 17 | // cc LoadWithTableDescriptorExample Load a coprocessor using the table descriptor 18 | // vv LoadWithTableDescriptorExample 19 | public class LoadWithTableDescriptorExample { 20 | 21 | public static void main(String[] args) throws IOException { 22 | Configuration conf = HBaseConfiguration.create(); 23 | Connection connection = ConnectionFactory.createConnection(conf); 24 | // ^^ LoadWithTableDescriptorExample 25 | HBaseHelper helper = HBaseHelper.getHelper(conf); 26 | helper.dropTable("testtable"); 27 | // vv LoadWithTableDescriptorExample 28 | TableName tableName = TableName.valueOf("testtable"); 29 | 30 | HTableDescriptor htd = new HTableDescriptor(tableName); // co LoadWithTableDescriptorExample-1-Define Define a table descriptor. 31 | htd.addFamily(new HColumnDescriptor("colfam1")); 32 | htd.setValue("COPROCESSOR$1", "|" + // co LoadWithTableDescriptorExample-2-AddCP Add the coprocessor definition to the descriptor, while omitting the path to the JAR file. 33 | RegionObserverExample.class.getCanonicalName() + 34 | "|" + Coprocessor.PRIORITY_USER); 35 | 36 | Admin admin = connection.getAdmin(); // co LoadWithTableDescriptorExample-3-Admin Acquire an administrative API to the cluster and add the table. 37 | admin.createTable(htd); 38 | 39 | System.out.println(admin.getTableDescriptor(tableName)); // co LoadWithTableDescriptorExample-4-Check Verify if the definition has been applied as expected. 40 | admin.close(); 41 | connection.close(); 42 | } 43 | } 44 | // ^^ LoadWithTableDescriptorExample 45 | -------------------------------------------------------------------------------- /ch04/src/main/java/coprocessor/LoadWithTableDescriptorExample2.java: -------------------------------------------------------------------------------- 1 | package coprocessor; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.conf.Configuration; 6 | import org.apache.hadoop.hbase.Coprocessor; 7 | import org.apache.hadoop.hbase.HBaseConfiguration; 8 | import org.apache.hadoop.hbase.HColumnDescriptor; 9 | import org.apache.hadoop.hbase.HTableDescriptor; 10 | import org.apache.hadoop.hbase.TableName; 11 | import org.apache.hadoop.hbase.client.Admin; 12 | import org.apache.hadoop.hbase.client.Connection; 13 | import org.apache.hadoop.hbase.client.ConnectionFactory; 14 | 15 | import util.HBaseHelper; 16 | 17 | // cc LoadWithTableDescriptorExample2 Load a coprocessor using the table descriptor using provided method 18 | public class LoadWithTableDescriptorExample2 { 19 | 20 | public static void main(String[] args) throws IOException { 21 | Configuration conf = HBaseConfiguration.create(); 22 | Connection connection = ConnectionFactory.createConnection(conf); 23 | HBaseHelper helper = HBaseHelper.getHelper(conf); 24 | helper.dropTable("testtable"); 25 | TableName tableName = TableName.valueOf("testtable"); 26 | 27 | // vv LoadWithTableDescriptorExample2 28 | HTableDescriptor htd = new HTableDescriptor(tableName); // co LoadWithTableDescriptorExample2-1-Create Use fluent interface to create and configure the instance. 29 | htd.addFamily(new HColumnDescriptor("colfam1")); 30 | /*[*/htd.addCoprocessor(RegionObserverExample.class.getCanonicalName(), 31 | null, Coprocessor.PRIORITY_USER, null);/*]*/ // co LoadWithTableDescriptorExample2-2-AddCP Use the provided method to add the coprocessor. 32 | 33 | Admin admin = connection.getAdmin(); 34 | admin.createTable(htd); 35 | // ^^ LoadWithTableDescriptorExample2 36 | 37 | System.out.println(admin.getTableDescriptor(tableName)); 38 | admin.close(); 39 | connection.close(); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /ch04/src/main/java/coprocessor/RegionObserverExample.java: -------------------------------------------------------------------------------- 1 | package coprocessor; 2 | 3 | import java.io.IOException; 4 | import java.util.List; 5 | 6 | import org.apache.commons.logging.Log; 7 | import org.apache.commons.logging.LogFactory; 8 | 9 | import org.apache.hadoop.hbase.Cell; 10 | import org.apache.hadoop.hbase.CellScanner; 11 | import org.apache.hadoop.hbase.client.Get; 12 | import org.apache.hadoop.hbase.client.Put; 13 | import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver; 14 | import org.apache.hadoop.hbase.coprocessor.ObserverContext; 15 | import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; 16 | import org.apache.hadoop.hbase.regionserver.HRegion; 17 | import org.apache.hadoop.hbase.util.Bytes; 18 | 19 | // cc RegionObserverExample Example region observer checking for special get requests 20 | // vv RegionObserverExample 21 | public class RegionObserverExample extends BaseRegionObserver { 22 | // ^^ RegionObserverExample 23 | public static final Log LOG = LogFactory.getLog(HRegion.class); 24 | // vv RegionObserverExample 25 | public static final byte[] FIXED_ROW = Bytes.toBytes("@@@GETTIME@@@"); 26 | 27 | @Override 28 | public void preGetOp(ObserverContext e, 29 | Get get, List results) throws IOException { 30 | // ^^ RegionObserverExample 31 | LOG.debug("Got preGet for row: " + Bytes.toStringBinary(get.getRow())); 32 | // vv RegionObserverExample 33 | if (Bytes.equals(get.getRow(), FIXED_ROW)) { // co RegionObserverExample-1-Check Check if the request row key matches a well known one. 34 | Put put = new Put(get.getRow()); 35 | put.addColumn(FIXED_ROW, FIXED_ROW, // co RegionObserverExample-2-Cell Create cell indirectly using a Put instance. 36 | Bytes.toBytes(System.currentTimeMillis())); 37 | CellScanner scanner = put.cellScanner(); 38 | scanner.advance(); 39 | Cell cell = scanner.current(); // co RegionObserverExample-3-Current Get first cell from Put using the CellScanner instance. 40 | // ^^ RegionObserverExample 41 | LOG.debug("Had a match, adding fake cell: " + cell); 42 | // vv RegionObserverExample 43 | results.add(cell); // co RegionObserverExample-4-Create Create a special KeyValue instance containing just the current time on the server. 44 | } 45 | } 46 | } 47 | // ^^ RegionObserverExample 48 | -------------------------------------------------------------------------------- /ch04/src/main/java/coprocessor/RegionObserverWithBypassExample.java: -------------------------------------------------------------------------------- 1 | package coprocessor; 2 | 3 | import java.io.IOException; 4 | import java.util.List; 5 | 6 | import org.apache.commons.logging.Log; 7 | import org.apache.commons.logging.LogFactory; 8 | import org.apache.hadoop.hbase.Cell; 9 | import org.apache.hadoop.hbase.CellUtil; 10 | import org.apache.hadoop.hbase.KeyValue; 11 | import org.apache.hadoop.hbase.client.Get; 12 | import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver; 13 | import org.apache.hadoop.hbase.coprocessor.ObserverContext; 14 | import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; 15 | import org.apache.hadoop.hbase.regionserver.HRegion; 16 | import org.apache.hadoop.hbase.util.Bytes; 17 | 18 | // cc RegionObserverWithBypassExample Example region observer checking for special get requests and bypassing further processing 19 | public class RegionObserverWithBypassExample extends BaseRegionObserver { 20 | public static final Log LOG = LogFactory.getLog(HRegion.class); 21 | public static final byte[] FIXED_ROW = Bytes.toBytes("@@@GETTIME@@@"); 22 | 23 | @Override 24 | public void preGetOp(ObserverContext e, Get get, 25 | List results) throws IOException { 26 | LOG.debug("Got preGet for row: " + Bytes.toStringBinary(get.getRow())); 27 | // vv RegionObserverWithBypassExample 28 | if (Bytes.equals(get.getRow(), FIXED_ROW)) { 29 | long time = System.currentTimeMillis(); 30 | Cell cell = CellUtil.createCell(get.getRow(), FIXED_ROW, FIXED_ROW, // co RegionObserverWithBypassExample-1-Cell Create cell directly using the supplied utility. 31 | time, KeyValue.Type.Put.getCode(), Bytes.toBytes(time)); 32 | // ^^ RegionObserverWithBypassExample 33 | LOG.debug("Had a match, adding fake cell: " + cell); 34 | // vv RegionObserverWithBypassExample 35 | results.add(cell); 36 | /*[*/e.bypass();/*]*/ // co RegionObserverWithBypassExample-2-Bypass Once the special cell is inserted all subsequent coprocessors are skipped. 37 | } 38 | // ^^ RegionObserverWithBypassExample 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /ch04/src/main/java/coprocessor/RegionObserverWithCompleteExample.java: -------------------------------------------------------------------------------- 1 | package coprocessor; 2 | 3 | import java.io.IOException; 4 | import java.util.List; 5 | 6 | import org.apache.commons.logging.Log; 7 | import org.apache.commons.logging.LogFactory; 8 | import org.apache.hadoop.hbase.Cell; 9 | import org.apache.hadoop.hbase.CellUtil; 10 | import org.apache.hadoop.hbase.KeyValue; 11 | import org.apache.hadoop.hbase.client.Get; 12 | import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver; 13 | import org.apache.hadoop.hbase.coprocessor.ObserverContext; 14 | import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; 15 | import org.apache.hadoop.hbase.regionserver.HRegion; 16 | import org.apache.hadoop.hbase.util.Bytes; 17 | 18 | // cc RegionObserverWithCompleteExample Example region observer checking for special get requests and bypassing all further processing 19 | public class RegionObserverWithCompleteExample extends BaseRegionObserver { 20 | public static final Log LOG = LogFactory.getLog(HRegion.class); 21 | public static final byte[] FIXED_ROW = Bytes.toBytes("@@@GETTIME@@@"); 22 | 23 | @Override 24 | public void preGetOp(ObserverContext e, Get get, 25 | List results) throws IOException { 26 | LOG.debug("Got preGet for row: " + Bytes.toStringBinary(get.getRow())); 27 | // vv RegionObserverWithCompleteExample 28 | if (Bytes.equals(get.getRow(), FIXED_ROW)) { 29 | long time = System.currentTimeMillis(); 30 | Cell cell = CellUtil.createCell(get.getRow(), FIXED_ROW, FIXED_ROW, 31 | time, KeyValue.Type.Put.getCode(), Bytes.toBytes(time)); 32 | // ^^ RegionObserverWithCompleteExample 33 | LOG.debug("Had a match, adding fake cell: " + cell); 34 | // vv RegionObserverWithCompleteExample 35 | results.add(cell); 36 | /*[*/e.complete();/*]*/ // co RegionObserverWithCompleteExample-2-Bypass Once the special cell is inserted all further processing is skipped. 37 | } 38 | // ^^ RegionObserverWithCompleteExample 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /ch04/src/main/java/coprocessor/RegionServerObserverExample.java: -------------------------------------------------------------------------------- 1 | package coprocessor; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.commons.logging.Log; 6 | import org.apache.commons.logging.LogFactory; 7 | import org.apache.hadoop.hbase.CellUtil; 8 | import org.apache.hadoop.hbase.HRegionInfo; 9 | import org.apache.hadoop.hbase.coprocessor.BaseRegionServerObserver; 10 | import org.apache.hadoop.hbase.coprocessor.ObserverContext; 11 | import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessorEnvironment; 12 | import org.apache.hadoop.hbase.regionserver.Region; 13 | import org.apache.hadoop.hbase.regionserver.wal.WALEdit; 14 | import org.apache.hadoop.hbase.wal.WALKey; 15 | 16 | // cc RegionServerObserverExample Example region server observer that ... 17 | public class RegionServerObserverExample extends BaseRegionServerObserver { 18 | public static final Log LOG = LogFactory.getLog(Region.class); 19 | 20 | @Override 21 | public void postMerge(ObserverContext c, 22 | Region regionA, Region regionB, Region mergedRegion) throws IOException { 23 | HRegionInfo regionInfo = mergedRegion.getRegionInfo(); 24 | // mergedRegion.getWAL().append(mergedRegion.getTableDesc(), regionInfo, 25 | // new WALKey(mergedRegion.getRegionName(), regionInfo.getTable()), 26 | // new WALEdit().add(CellUtil.createCell())) 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /ch04/src/main/java/filters/ColumnPaginationFilterExample.java: -------------------------------------------------------------------------------- 1 | package filters; 2 | 3 | // cc ColumnPaginationFilterExample Example paginating through columns in a row 4 | import java.io.IOException; 5 | 6 | import org.apache.hadoop.conf.Configuration; 7 | import org.apache.hadoop.hbase.HBaseConfiguration; 8 | import org.apache.hadoop.hbase.TableName; 9 | import org.apache.hadoop.hbase.client.Connection; 10 | import org.apache.hadoop.hbase.client.ConnectionFactory; 11 | import org.apache.hadoop.hbase.client.Result; 12 | import org.apache.hadoop.hbase.client.ResultScanner; 13 | import org.apache.hadoop.hbase.client.Scan; 14 | import org.apache.hadoop.hbase.client.Table; 15 | import org.apache.hadoop.hbase.filter.ColumnPaginationFilter; 16 | import org.apache.hadoop.hbase.filter.Filter; 17 | 18 | import util.HBaseHelper; 19 | 20 | public class ColumnPaginationFilterExample { 21 | 22 | public static void main(String[] args) throws IOException { 23 | Configuration conf = HBaseConfiguration.create(); 24 | 25 | HBaseHelper helper = HBaseHelper.getHelper(conf); 26 | helper.dropTable("testtable"); 27 | helper.createTable("testtable", "colfam1"); 28 | System.out.println("Adding rows to table..."); 29 | helper.fillTable("testtable", 1, 10, 30, 2, true, "colfam1"); 30 | 31 | Connection connection = ConnectionFactory.createConnection(conf); 32 | Table table = connection.getTable(TableName.valueOf("testtable")); 33 | // vv ColumnPaginationFilterExample 34 | Filter filter = new ColumnPaginationFilter(5, 15); 35 | 36 | Scan scan = new Scan(); 37 | scan.setFilter(filter); 38 | ResultScanner scanner = table.getScanner(scan); 39 | // ^^ ColumnPaginationFilterExample 40 | System.out.println("Results of scan:"); 41 | // vv ColumnPaginationFilterExample 42 | for (Result result : scanner) { 43 | System.out.println(result); 44 | } 45 | scanner.close(); 46 | // ^^ ColumnPaginationFilterExample 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /ch04/src/main/java/filters/ColumnPrefixFilterExample.java: -------------------------------------------------------------------------------- 1 | package filters; 2 | 3 | // cc ColumnPrefixFilterExample Example filtering by column prefix 4 | import java.io.IOException; 5 | 6 | import org.apache.hadoop.conf.Configuration; 7 | import org.apache.hadoop.hbase.HBaseConfiguration; 8 | import org.apache.hadoop.hbase.TableName; 9 | import org.apache.hadoop.hbase.client.Connection; 10 | import org.apache.hadoop.hbase.client.ConnectionFactory; 11 | import org.apache.hadoop.hbase.client.Result; 12 | import org.apache.hadoop.hbase.client.ResultScanner; 13 | import org.apache.hadoop.hbase.client.Scan; 14 | import org.apache.hadoop.hbase.client.Table; 15 | import org.apache.hadoop.hbase.filter.ColumnPrefixFilter; 16 | import org.apache.hadoop.hbase.filter.Filter; 17 | 18 | import org.apache.hadoop.hbase.util.Bytes; 19 | import util.HBaseHelper; 20 | 21 | public class ColumnPrefixFilterExample { 22 | 23 | public static void main(String[] args) throws IOException { 24 | Configuration conf = HBaseConfiguration.create(); 25 | 26 | HBaseHelper helper = HBaseHelper.getHelper(conf); 27 | helper.dropTable("testtable"); 28 | helper.createTable("testtable", "colfam1"); 29 | System.out.println("Adding rows to table..."); 30 | helper.fillTable("testtable", 1, 10, 30, 0, true, "colfam1"); 31 | 32 | Connection connection = ConnectionFactory.createConnection(conf); 33 | Table table = connection.getTable(TableName.valueOf("testtable")); 34 | // vv ColumnPrefixFilterExample 35 | Filter filter = new ColumnPrefixFilter(Bytes.toBytes("col-1")); 36 | 37 | Scan scan = new Scan(); 38 | scan.setFilter(filter); 39 | ResultScanner scanner = table.getScanner(scan); 40 | // ^^ ColumnPrefixFilterExample 41 | System.out.println("Results of scan:"); 42 | // vv ColumnPrefixFilterExample 43 | for (Result result : scanner) { 44 | System.out.println(result); 45 | } 46 | scanner.close(); 47 | // ^^ ColumnPrefixFilterExample 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /ch04/src/main/java/filters/ColumnRangeFilterExample.java: -------------------------------------------------------------------------------- 1 | package filters; 2 | 3 | // cc ColumnRangeFilterExample Example filtering by columns within a given range 4 | import java.io.IOException; 5 | 6 | import org.apache.hadoop.conf.Configuration; 7 | import org.apache.hadoop.hbase.HBaseConfiguration; 8 | import org.apache.hadoop.hbase.TableName; 9 | import org.apache.hadoop.hbase.client.Connection; 10 | import org.apache.hadoop.hbase.client.ConnectionFactory; 11 | import org.apache.hadoop.hbase.client.Result; 12 | import org.apache.hadoop.hbase.client.ResultScanner; 13 | import org.apache.hadoop.hbase.client.Scan; 14 | import org.apache.hadoop.hbase.client.Table; 15 | import org.apache.hadoop.hbase.filter.ColumnRangeFilter; 16 | import org.apache.hadoop.hbase.filter.Filter; 17 | import org.apache.hadoop.hbase.util.Bytes; 18 | 19 | import util.HBaseHelper; 20 | 21 | public class ColumnRangeFilterExample { 22 | 23 | public static void main(String[] args) throws IOException { 24 | Configuration conf = HBaseConfiguration.create(); 25 | 26 | HBaseHelper helper = HBaseHelper.getHelper(conf); 27 | helper.dropTable("testtable"); 28 | helper.createTable("testtable", "colfam1"); 29 | System.out.println("Adding rows to table..."); 30 | helper.fillTable("testtable", 1, 10, 30, 2, true, "colfam1"); 31 | 32 | Connection connection = ConnectionFactory.createConnection(conf); 33 | Table table = connection.getTable(TableName.valueOf("testtable")); 34 | // vv ColumnRangeFilterExample 35 | Filter filter = new ColumnRangeFilter(Bytes.toBytes("col-05"), true, 36 | Bytes.toBytes("col-11"), false); 37 | 38 | Scan scan = new Scan() 39 | .setStartRow(Bytes.toBytes("row-03")) 40 | .setStopRow(Bytes.toBytes("row-05")) 41 | .setFilter(filter); 42 | ResultScanner scanner = table.getScanner(scan); 43 | // ^^ ColumnRangeFilterExample 44 | System.out.println("Results of scan:"); 45 | // vv ColumnRangeFilterExample 46 | for (Result result : scanner) { 47 | System.out.println(result); 48 | } 49 | scanner.close(); 50 | // ^^ ColumnRangeFilterExample 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /ch04/src/main/java/filters/FirstKeyOnlyFilterExample.java: -------------------------------------------------------------------------------- 1 | package filters; 2 | 3 | // cc FirstKeyOnlyFilterExample Only returns the first found cell from each row 4 | import java.io.IOException; 5 | 6 | import org.apache.hadoop.conf.Configuration; 7 | import org.apache.hadoop.hbase.Cell; 8 | import org.apache.hadoop.hbase.HBaseConfiguration; 9 | import org.apache.hadoop.hbase.TableName; 10 | import org.apache.hadoop.hbase.client.Connection; 11 | import org.apache.hadoop.hbase.client.ConnectionFactory; 12 | import org.apache.hadoop.hbase.client.Result; 13 | import org.apache.hadoop.hbase.client.ResultScanner; 14 | import org.apache.hadoop.hbase.client.Scan; 15 | import org.apache.hadoop.hbase.client.Table; 16 | import org.apache.hadoop.hbase.filter.Filter; 17 | import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; 18 | import org.apache.hadoop.hbase.util.Bytes; 19 | 20 | import util.HBaseHelper; 21 | 22 | public class FirstKeyOnlyFilterExample { 23 | 24 | public static void main(String[] args) throws IOException { 25 | Configuration conf = HBaseConfiguration.create(); 26 | 27 | HBaseHelper helper = HBaseHelper.getHelper(conf); 28 | helper.dropTable("testtable"); 29 | helper.createTable("testtable", "colfam1"); 30 | System.out.println("Adding rows to table..."); 31 | helper.fillTableRandom("testtable", /* row */ 1, 30, 0, 32 | /* col */ 1, 30, 0, /* val */ 0, 100, 0, true, "colfam1"); 33 | 34 | Connection connection = ConnectionFactory.createConnection(conf); 35 | Table table = connection.getTable(TableName.valueOf("testtable")); 36 | // vv FirstKeyOnlyFilterExample 37 | Filter filter = new FirstKeyOnlyFilter(); 38 | 39 | Scan scan = new Scan(); 40 | scan.setFilter(filter); 41 | ResultScanner scanner = table.getScanner(scan); 42 | // ^^ FirstKeyOnlyFilterExample 43 | System.out.println("Results of scan:"); 44 | // vv FirstKeyOnlyFilterExample 45 | int rowCount = 0; 46 | for (Result result : scanner) { 47 | for (Cell cell : result.rawCells()) { 48 | System.out.println("Cell: " + cell + ", Value: " + 49 | Bytes.toString(cell.getValueArray(), cell.getValueOffset(), 50 | cell.getValueLength())); 51 | } 52 | rowCount++; 53 | } 54 | System.out.println("Total num of rows: " + rowCount); 55 | scanner.close(); 56 | // ^^ FirstKeyOnlyFilterExample 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /ch04/src/main/java/filters/FuzzyRowFilterExample.java: -------------------------------------------------------------------------------- 1 | package filters; 2 | 3 | // cc FuzzyRowFilterExample Example filtering by column prefix 4 | import java.io.IOException; 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | 8 | import org.apache.hadoop.conf.Configuration; 9 | import org.apache.hadoop.hbase.HBaseConfiguration; 10 | import org.apache.hadoop.hbase.TableName; 11 | import org.apache.hadoop.hbase.client.Connection; 12 | import org.apache.hadoop.hbase.client.ConnectionFactory; 13 | import org.apache.hadoop.hbase.client.Result; 14 | import org.apache.hadoop.hbase.client.ResultScanner; 15 | import org.apache.hadoop.hbase.client.Scan; 16 | import org.apache.hadoop.hbase.client.Table; 17 | import org.apache.hadoop.hbase.filter.Filter; 18 | import org.apache.hadoop.hbase.filter.FuzzyRowFilter; 19 | import org.apache.hadoop.hbase.util.Bytes; 20 | 21 | import org.apache.hadoop.hbase.util.Pair; 22 | import util.HBaseHelper; 23 | 24 | public class FuzzyRowFilterExample { 25 | 26 | public static void main(String[] args) throws IOException { 27 | Configuration conf = HBaseConfiguration.create(); 28 | 29 | HBaseHelper helper = HBaseHelper.getHelper(conf); 30 | helper.dropTable("testtable"); 31 | helper.createTable("testtable", "colfam1"); 32 | System.out.println("Adding rows to table..."); 33 | helper.fillTable("testtable", 1, 20, 10, 2, true, "colfam1"); 34 | 35 | Connection connection = ConnectionFactory.createConnection(conf); 36 | Table table = connection.getTable(TableName.valueOf("testtable")); 37 | // vv FuzzyRowFilterExample 38 | List> keys = new ArrayList>(); 39 | keys.add(new Pair( 40 | Bytes.toBytes("row-?5"), new byte[] { 0, 0, 0, 0, 1, 0 })); 41 | Filter filter = new FuzzyRowFilter(keys); 42 | 43 | Scan scan = new Scan() 44 | .addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("col-5")) 45 | .setFilter(filter); 46 | ResultScanner scanner = table.getScanner(scan); 47 | // ^^ FuzzyRowFilterExample 48 | System.out.println("Results of scan:"); 49 | // vv FuzzyRowFilterExample 50 | for (Result result : scanner) { 51 | System.out.println(result); 52 | } 53 | scanner.close(); 54 | // ^^ FuzzyRowFilterExample 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /ch04/src/main/java/filters/InclusiveStopFilterExample.java: -------------------------------------------------------------------------------- 1 | package filters; 2 | 3 | // cc InclusiveStopFilterExample Example using a filter to include a stop row 4 | import java.io.IOException; 5 | 6 | import org.apache.hadoop.conf.Configuration; 7 | import org.apache.hadoop.hbase.HBaseConfiguration; 8 | import org.apache.hadoop.hbase.TableName; 9 | import org.apache.hadoop.hbase.client.Connection; 10 | import org.apache.hadoop.hbase.client.ConnectionFactory; 11 | import org.apache.hadoop.hbase.client.Result; 12 | import org.apache.hadoop.hbase.client.ResultScanner; 13 | import org.apache.hadoop.hbase.client.Scan; 14 | import org.apache.hadoop.hbase.client.Table; 15 | import org.apache.hadoop.hbase.filter.Filter; 16 | import org.apache.hadoop.hbase.filter.InclusiveStopFilter; 17 | import org.apache.hadoop.hbase.util.Bytes; 18 | 19 | import util.HBaseHelper; 20 | 21 | public class InclusiveStopFilterExample { 22 | 23 | public static void main(String[] args) throws IOException { 24 | Configuration conf = HBaseConfiguration.create(); 25 | 26 | HBaseHelper helper = HBaseHelper.getHelper(conf); 27 | helper.dropTable("testtable"); 28 | helper.createTable("testtable", "colfam1"); 29 | System.out.println("Adding rows to table..."); 30 | helper.fillTable("testtable", 1, 100, 1, "colfam1"); 31 | 32 | Connection connection = ConnectionFactory.createConnection(conf); 33 | Table table = connection.getTable(TableName.valueOf("testtable")); 34 | // vv InclusiveStopFilterExample 35 | Filter filter = new InclusiveStopFilter(Bytes.toBytes("row-5")); 36 | 37 | Scan scan = new Scan(); 38 | scan.setStartRow(Bytes.toBytes("row-3")); 39 | scan.setFilter(filter); 40 | ResultScanner scanner = table.getScanner(scan); 41 | // ^^ InclusiveStopFilterExample 42 | System.out.println("Results of scan:"); 43 | // vv InclusiveStopFilterExample 44 | for (Result result : scanner) { 45 | System.out.println(result); 46 | } 47 | scanner.close(); 48 | // ^^ InclusiveStopFilterExample 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /ch04/src/main/java/filters/MultipleColumnPrefixFilterExample.java: -------------------------------------------------------------------------------- 1 | package filters; 2 | 3 | // cc MultipleColumnPrefixFilterExample Example filtering by column prefix 4 | import java.io.IOException; 5 | 6 | import org.apache.hadoop.conf.Configuration; 7 | import org.apache.hadoop.hbase.Cell; 8 | import org.apache.hadoop.hbase.HBaseConfiguration; 9 | import org.apache.hadoop.hbase.TableName; 10 | import org.apache.hadoop.hbase.client.Connection; 11 | import org.apache.hadoop.hbase.client.ConnectionFactory; 12 | import org.apache.hadoop.hbase.client.Result; 13 | import org.apache.hadoop.hbase.client.ResultScanner; 14 | import org.apache.hadoop.hbase.client.Scan; 15 | import org.apache.hadoop.hbase.client.Table; 16 | import org.apache.hadoop.hbase.filter.Filter; 17 | import org.apache.hadoop.hbase.filter.MultipleColumnPrefixFilter; 18 | import org.apache.hadoop.hbase.util.Bytes; 19 | 20 | import util.HBaseHelper; 21 | 22 | public class MultipleColumnPrefixFilterExample { 23 | 24 | public static void main(String[] args) throws IOException { 25 | Configuration conf = HBaseConfiguration.create(); 26 | 27 | HBaseHelper helper = HBaseHelper.getHelper(conf); 28 | helper.dropTable("testtable"); 29 | helper.createTable("testtable", "colfam1"); 30 | System.out.println("Adding rows to table..."); 31 | helper.fillTable("testtable", 1, 30, 50, 0, true, "colfam1"); 32 | 33 | Connection connection = ConnectionFactory.createConnection(conf); 34 | Table table = connection.getTable(TableName.valueOf("testtable")); 35 | // vv MultipleColumnPrefixFilterExample 36 | Filter filter = new MultipleColumnPrefixFilter(new byte[][] { 37 | Bytes.toBytes("col-1"), Bytes.toBytes("col-2") 38 | }); 39 | 40 | Scan scan = new Scan() 41 | .setRowPrefixFilter(Bytes.toBytes("row-1")) // co MultipleColumnPrefixFilterExample-1-Row Limit to rows starting with a specific prefix. 42 | .setFilter(filter); 43 | ResultScanner scanner = table.getScanner(scan); 44 | // ^^ MultipleColumnPrefixFilterExample 45 | System.out.println("Results of scan:"); 46 | // vv MultipleColumnPrefixFilterExample 47 | for (Result result : scanner) { 48 | System.out.print(Bytes.toString(result.getRow()) + ": "); 49 | for (Cell cell : result.rawCells()) { 50 | System.out.print(Bytes.toString(cell.getQualifierArray(), 51 | cell.getQualifierOffset(), cell.getQualifierLength()) + ", "); 52 | } 53 | System.out.println(); 54 | } 55 | scanner.close(); 56 | // ^^ MultipleColumnPrefixFilterExample 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /ch04/src/main/java/filters/PageFilterExample.java: -------------------------------------------------------------------------------- 1 | package filters; 2 | 3 | // cc PageFilterExample Example using a filter to paginate through rows 4 | import java.io.IOException; 5 | 6 | import org.apache.hadoop.conf.Configuration; 7 | import org.apache.hadoop.hbase.HBaseConfiguration; 8 | import org.apache.hadoop.hbase.TableName; 9 | import org.apache.hadoop.hbase.client.Connection; 10 | import org.apache.hadoop.hbase.client.ConnectionFactory; 11 | import org.apache.hadoop.hbase.client.Result; 12 | import org.apache.hadoop.hbase.client.ResultScanner; 13 | import org.apache.hadoop.hbase.client.Scan; 14 | import org.apache.hadoop.hbase.client.Table; 15 | import org.apache.hadoop.hbase.filter.Filter; 16 | import org.apache.hadoop.hbase.filter.PageFilter; 17 | import org.apache.hadoop.hbase.util.Bytes; 18 | 19 | import util.HBaseHelper; 20 | 21 | public class PageFilterExample { 22 | 23 | // vv PageFilterExample 24 | private static final byte[] POSTFIX = new byte[] { 0x00 }; 25 | // ^^ PageFilterExample 26 | 27 | public static void main(String[] args) throws IOException { 28 | Configuration conf = HBaseConfiguration.create(); 29 | 30 | HBaseHelper helper = HBaseHelper.getHelper(conf); 31 | helper.dropTable("testtable"); 32 | helper.createTable("testtable", "colfam1"); 33 | System.out.println("Adding rows to table..."); 34 | helper.fillTable("testtable", 1, 1000, 10, "colfam1"); 35 | 36 | Connection connection = ConnectionFactory.createConnection(conf); 37 | Table table = connection.getTable(TableName.valueOf("testtable")); 38 | 39 | // vv PageFilterExample 40 | Filter filter = new PageFilter(15); 41 | 42 | int totalRows = 0; 43 | byte[] lastRow = null; 44 | while (true) { 45 | Scan scan = new Scan(); 46 | scan.setFilter(filter); 47 | if (lastRow != null) { 48 | byte[] startRow = Bytes.add(lastRow, POSTFIX); 49 | System.out.println("start row: " + 50 | Bytes.toStringBinary(startRow)); 51 | scan.setStartRow(startRow); 52 | } 53 | ResultScanner scanner = table.getScanner(scan); 54 | int localRows = 0; 55 | Result result; 56 | while ((result = scanner.next()) != null) { 57 | System.out.println(localRows++ + ": " + result); 58 | totalRows++; 59 | lastRow = result.getRow(); 60 | } 61 | scanner.close(); 62 | if (localRows == 0) break; 63 | } 64 | System.out.println("total rows: " + totalRows); 65 | // ^^ PageFilterExample 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /ch04/src/main/java/filters/QualifierFilterExample.java: -------------------------------------------------------------------------------- 1 | package filters; 2 | 3 | // cc QualifierFilterExample Example using a filter to include only specific column qualifiers 4 | import java.io.IOException; 5 | 6 | import org.apache.hadoop.conf.Configuration; 7 | import org.apache.hadoop.hbase.HBaseConfiguration; 8 | import org.apache.hadoop.hbase.TableName; 9 | import org.apache.hadoop.hbase.client.Connection; 10 | import org.apache.hadoop.hbase.client.ConnectionFactory; 11 | import org.apache.hadoop.hbase.client.Get; 12 | import org.apache.hadoop.hbase.client.Result; 13 | import org.apache.hadoop.hbase.client.ResultScanner; 14 | import org.apache.hadoop.hbase.client.Scan; 15 | import org.apache.hadoop.hbase.client.Table; 16 | import org.apache.hadoop.hbase.filter.BinaryComparator; 17 | import org.apache.hadoop.hbase.filter.CompareFilter; 18 | import org.apache.hadoop.hbase.filter.Filter; 19 | import org.apache.hadoop.hbase.filter.QualifierFilter; 20 | import org.apache.hadoop.hbase.util.Bytes; 21 | 22 | import util.HBaseHelper; 23 | 24 | public class QualifierFilterExample { 25 | 26 | public static void main(String[] args) throws IOException { 27 | Configuration conf = HBaseConfiguration.create(); 28 | 29 | HBaseHelper helper = HBaseHelper.getHelper(conf); 30 | helper.dropTable("testtable"); 31 | helper.createTable("testtable", "colfam1", "colfam2"); 32 | System.out.println("Adding rows to table..."); 33 | helper.fillTable("testtable", 1, 10, 10, "colfam1", "colfam2"); 34 | 35 | Connection connection = ConnectionFactory.createConnection(conf); 36 | Table table = connection.getTable(TableName.valueOf("testtable")); 37 | 38 | // vv QualifierFilterExample 39 | Filter filter = new QualifierFilter(CompareFilter.CompareOp.LESS_OR_EQUAL, 40 | new BinaryComparator(Bytes.toBytes("col-2"))); 41 | 42 | Scan scan = new Scan(); 43 | scan.setFilter(filter); 44 | ResultScanner scanner = table.getScanner(scan); 45 | // ^^ QualifierFilterExample 46 | System.out.println("Scanning table... "); 47 | // vv QualifierFilterExample 48 | for (Result result : scanner) { 49 | System.out.println(result); 50 | } 51 | scanner.close(); 52 | 53 | Get get = new Get(Bytes.toBytes("row-5")); 54 | get.setFilter(filter); 55 | Result result = table.get(get); 56 | System.out.println("Result of get(): " + result); 57 | // ^^ QualifierFilterExample 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /ch04/src/main/java/filters/RandomRowFilterExample.java: -------------------------------------------------------------------------------- 1 | package filters; 2 | 3 | // cc RandomRowFilterExample Example filtering rows randomly 4 | import java.io.IOException; 5 | 6 | import org.apache.hadoop.conf.Configuration; 7 | import org.apache.hadoop.hbase.HBaseConfiguration; 8 | import org.apache.hadoop.hbase.TableName; 9 | import org.apache.hadoop.hbase.client.Connection; 10 | import org.apache.hadoop.hbase.client.ConnectionFactory; 11 | import org.apache.hadoop.hbase.client.Result; 12 | import org.apache.hadoop.hbase.client.ResultScanner; 13 | import org.apache.hadoop.hbase.client.Scan; 14 | import org.apache.hadoop.hbase.client.Table; 15 | import org.apache.hadoop.hbase.filter.Filter; 16 | import org.apache.hadoop.hbase.filter.RandomRowFilter; 17 | import org.apache.hadoop.hbase.util.Bytes; 18 | 19 | import util.HBaseHelper; 20 | 21 | public class RandomRowFilterExample { 22 | 23 | public static void main(String[] args) throws IOException { 24 | Configuration conf = HBaseConfiguration.create(); 25 | 26 | HBaseHelper helper = HBaseHelper.getHelper(conf); 27 | helper.dropTable("testtable"); 28 | helper.createTable("testtable", "colfam1"); 29 | System.out.println("Adding rows to table..."); 30 | helper.fillTable("testtable", 1, 10, 30, 0, true, "colfam1"); 31 | 32 | Connection connection = ConnectionFactory.createConnection(conf); 33 | Table table = connection.getTable(TableName.valueOf("testtable")); 34 | // vv RandomRowFilterExample 35 | Filter filter = new RandomRowFilter(0.5f); 36 | 37 | for (int loop = 1; loop <= 3; loop++) { 38 | Scan scan = new Scan(); 39 | scan.setFilter(filter); 40 | ResultScanner scanner = table.getScanner(scan); 41 | // ^^ RandomRowFilterExample 42 | System.out.println("Results of scan for loop: " + loop); 43 | // vv RandomRowFilterExample 44 | for (Result result : scanner) { 45 | System.out.println(Bytes.toString(result.getRow())); 46 | } 47 | scanner.close(); 48 | } 49 | // ^^ RandomRowFilterExample 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /ch04/src/main/protobuf/CustomFilters.proto: -------------------------------------------------------------------------------- 1 | option java_package = "filters.generated"; 2 | option java_outer_classname = "FilterProtos"; 3 | option java_generic_services = true; 4 | option java_generate_equals_and_hash = true; 5 | option optimize_for = SPEED; 6 | 7 | message CustomFilter { 8 | required bytes value = 1; 9 | } -------------------------------------------------------------------------------- /ch04/src/main/protobuf/ObserverStatisticsService.proto: -------------------------------------------------------------------------------- 1 | option java_package = "coprocessor.generated"; 2 | option java_outer_classname = "ObserverStatisticsProtos"; 3 | option java_generic_services = true; 4 | option java_generate_equals_and_hash = true; 5 | option optimize_for = SPEED; 6 | 7 | message NameInt32Pair { 8 | optional string name = 1; 9 | optional int32 value = 2; 10 | } 11 | 12 | message StatisticsRequest { 13 | optional bool clear = 1 [default = false]; 14 | } 15 | 16 | message StatisticsResponse { 17 | repeated NameInt32Pair attribute = 1; 18 | } 19 | 20 | service ObserverStatisticsService { 21 | rpc getStatistics(StatisticsRequest) 22 | returns (StatisticsResponse); 23 | } 24 | -------------------------------------------------------------------------------- /ch04/src/main/protobuf/RowCountService.proto: -------------------------------------------------------------------------------- 1 | option java_package = "coprocessor.generated"; 2 | option java_outer_classname = "RowCounterProtos"; 3 | option java_generic_services = true; 4 | option java_generate_equals_and_hash = true; 5 | option optimize_for = SPEED; 6 | 7 | message CountRequest { 8 | } 9 | 10 | message CountResponse { 11 | required int64 count = 1 [default = 0]; 12 | } 13 | 14 | service RowCountService { 15 | rpc getRowCount(CountRequest) 16 | returns (CountResponse); 17 | rpc getCellCount(CountRequest) 18 | returns (CountResponse); 19 | } 20 | 21 | 22 | /* 23 | public interface RowCountProtocol extends CoprocessorProtocol { 24 | long getRowCount() throws IOException; 25 | long getRowCount(Filter filter) throws IOException; 26 | long getKeyValueCount() throws IOException; 27 | } 28 | */ 29 | -------------------------------------------------------------------------------- /ch04/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO, stdout 2 | 3 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 4 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 5 | 6 | # Pattern to output the caller's file name and line number. 7 | log4j.appender.stdout.layout.ConversionPattern=%5p [%t] (%F:%L) - %m%n 8 | 9 | log4j.logger.org.apache=WARN -------------------------------------------------------------------------------- /ch05/bin/run.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | # 3 | # run.sh: helps executing the examples by setting up the Java CLASSPATH. 4 | # 5 | 6 | # get the current directory 7 | bin=`dirname "$0"` 8 | bin=`cd "$bin">/dev/null; pwd` 9 | 10 | # check command line args 11 | if [[ $# == 0 ]]; then 12 | echo "usage: $(basename $0) " 13 | exit 1; 14 | fi 15 | 16 | # set up Maven environment 17 | MVN="mvn" 18 | if [ "$MAVEN_HOME" != "" ]; then 19 | MVN=${MAVEN_HOME}/bin/mvn 20 | fi 21 | 22 | # classpath initially contains $HBASE_CONF_DIR 23 | CLASSPATH="${HBASE_CONF_DIR}" 24 | 25 | # add classes first, triggers log4j.properties priority 26 | if [ -d "${bin}/../target/classes" ]; then 27 | CLASSPATH=${CLASSPATH}:${bin}/../target/classes 28 | fi 29 | 30 | # create and cache Maven classpath 31 | cpfile="${bin}/../target/cached_classpath.txt" 32 | if [ ! -f "${cpfile}" ]; then 33 | ${MVN} -f "${bin}/../pom.xml" dependency:build-classpath -Dmdep.outputFile="${cpfile}" &> /dev/null 34 | fi 35 | CLASSPATH=${CLASSPATH}:`cat "${cpfile}"` 36 | 37 | JAVA=$JAVA_HOME/bin/java 38 | JAVA_HEAP_MAX=-Xmx512m 39 | 40 | "$JAVA" $JAVA_HEAP_MAX -classpath "$CLASSPATH" "$@" -------------------------------------------------------------------------------- /ch05/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | 6 | com.larsgeorge 7 | hbase-book 8 | 2.0 9 | ../pom.xml 10 | 11 | 12 | com.larsgeorge 13 | hbase-book-ch05 14 | 2.0 15 | jar 16 | 17 | HBase Book Chapter 5 18 | 19 | 20 | 21 | com.larsgeorge 22 | hbase-book-common 23 | ${project.version} 24 | 25 | 26 | -------------------------------------------------------------------------------- /ch05/src/main/java/admin/CreateTableExample.java: -------------------------------------------------------------------------------- 1 | package admin; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.conf.Configuration; 6 | import org.apache.hadoop.hbase.HBaseConfiguration; 7 | import org.apache.hadoop.hbase.HColumnDescriptor; 8 | import org.apache.hadoop.hbase.HTableDescriptor; 9 | import org.apache.hadoop.hbase.TableName; 10 | import org.apache.hadoop.hbase.client.Admin; 11 | import org.apache.hadoop.hbase.client.Connection; 12 | import org.apache.hadoop.hbase.client.ConnectionFactory; 13 | import org.apache.hadoop.hbase.util.Bytes; 14 | 15 | import util.HBaseHelper; 16 | 17 | // cc CreateTableExample Example using the administrative API to create a table 18 | public class CreateTableExample { 19 | 20 | public static void main(String[] args) throws IOException, InterruptedException { 21 | // vv CreateTableExample 22 | Configuration conf = HBaseConfiguration.create(); 23 | // ^^ CreateTableExample 24 | HBaseHelper helper = HBaseHelper.getHelper(conf); 25 | helper.dropTable("testtable"); 26 | // vv CreateTableExample 27 | Connection connection = ConnectionFactory.createConnection(conf); 28 | Admin admin = connection.getAdmin(); // co CreateTableExample-1-CreateAdmin Create a administrative API instance. 29 | 30 | TableName tableName = TableName.valueOf("testtable"); 31 | HTableDescriptor desc = new HTableDescriptor(tableName); // co CreateTableExample-2-CreateHTD Create the table descriptor instance. 32 | 33 | HColumnDescriptor coldef = new HColumnDescriptor( // co CreateTableExample-3-CreateHCD Create a column family descriptor and add it to the table descriptor. 34 | Bytes.toBytes("colfam1")); 35 | desc.addFamily(coldef); 36 | 37 | admin.createTable(desc); // co CreateTableExample-4-CreateTable Call the createTable() method to do the actual work. 38 | 39 | boolean avail = admin.isTableAvailable(tableName); // co CreateTableExample-5-Check Check if the table is available. 40 | System.out.println("Table available: " + avail); 41 | // ^^ CreateTableExample 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /ch05/src/main/java/admin/CreateTableWithNamespaceExample.java: -------------------------------------------------------------------------------- 1 | package admin; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.conf.Configuration; 6 | import org.apache.hadoop.hbase.HBaseConfiguration; 7 | import org.apache.hadoop.hbase.HColumnDescriptor; 8 | import org.apache.hadoop.hbase.HTableDescriptor; 9 | import org.apache.hadoop.hbase.NamespaceDescriptor; 10 | import org.apache.hadoop.hbase.TableName; 11 | import org.apache.hadoop.hbase.client.Admin; 12 | import org.apache.hadoop.hbase.client.Connection; 13 | import org.apache.hadoop.hbase.client.ConnectionFactory; 14 | import org.apache.hadoop.hbase.util.Bytes; 15 | 16 | import util.HBaseHelper; 17 | 18 | // cc CreateTableWithNamespaceExample Example using the administrative API to create a table with a custom namespace 19 | public class CreateTableWithNamespaceExample { 20 | 21 | public static void main(String[] args) throws IOException, InterruptedException { 22 | Configuration conf = HBaseConfiguration.create(); 23 | HBaseHelper helper = HBaseHelper.getHelper(conf); 24 | helper.dropTable("testtable"); 25 | Connection connection = ConnectionFactory.createConnection(conf); 26 | Admin admin = connection.getAdmin(); 27 | 28 | // vv CreateTableWithNamespaceExample 29 | /*[*/NamespaceDescriptor namespace = 30 | NamespaceDescriptor.create("testspace").build(); 31 | admin.createNamespace(namespace);/*]*/ 32 | 33 | TableName tableName = TableName.valueOf("testspace", "testtable"); 34 | HTableDescriptor desc = new HTableDescriptor(tableName); 35 | 36 | HColumnDescriptor coldef = new HColumnDescriptor( 37 | Bytes.toBytes("colfam1")); 38 | desc.addFamily(coldef); 39 | 40 | admin.createTable(desc); 41 | // ^^ CreateTableWithNamespaceExample 42 | 43 | boolean avail = admin.isTableAvailable(tableName); 44 | System.out.println("Table available: " + avail); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /ch05/src/main/java/admin/HColumnDescriptorExample.java: -------------------------------------------------------------------------------- 1 | package admin; 2 | 3 | import org.apache.hadoop.hbase.HColumnDescriptor; 4 | import org.apache.hadoop.hbase.io.ImmutableBytesWritable; 5 | import org.apache.hadoop.hbase.regionserver.BloomType; 6 | import org.apache.hadoop.hbase.util.Bytes; 7 | 8 | import java.io.IOException; 9 | import java.util.Map; 10 | 11 | // cc HColumnDescriptorExample Example how to create a HColumnDescriptor in code 12 | public class HColumnDescriptorExample { 13 | 14 | public static void main(String[] args) throws IOException, InterruptedException { 15 | // vv HColumnDescriptorExample 16 | HColumnDescriptor desc = new HColumnDescriptor("colfam1") 17 | .setValue("test-key", "test-value") 18 | .setBloomFilterType(BloomType.ROWCOL); 19 | 20 | System.out.println("Column Descriptor: " + desc); 21 | 22 | System.out.print("Values: "); 23 | for (Map.Entry 24 | entry : desc.getValues().entrySet()) { 25 | System.out.print(Bytes.toString(entry.getKey().get()) + 26 | " -> " + Bytes.toString(entry.getValue().get()) + ", "); 27 | } 28 | System.out.println(); 29 | 30 | System.out.println("Defaults: " + 31 | HColumnDescriptor.getDefaultValues()); 32 | 33 | System.out.println("Custom: " + 34 | desc.toStringCustomizedValues()); 35 | 36 | System.out.println("Units:"); 37 | System.out.println(HColumnDescriptor.TTL + " -> " + 38 | desc.getUnit(HColumnDescriptor.TTL)); 39 | System.out.println(HColumnDescriptor.BLOCKSIZE + " -> " + 40 | desc.getUnit(HColumnDescriptor.BLOCKSIZE)); 41 | // ^^ HColumnDescriptorExample 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /ch05/src/main/java/admin/ListTablesExample.java: -------------------------------------------------------------------------------- 1 | package admin; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.conf.Configuration; 6 | import org.apache.hadoop.hbase.HBaseConfiguration; 7 | import org.apache.hadoop.hbase.HTableDescriptor; 8 | import org.apache.hadoop.hbase.TableName; 9 | import org.apache.hadoop.hbase.client.Admin; 10 | import org.apache.hadoop.hbase.client.Connection; 11 | import org.apache.hadoop.hbase.client.ConnectionFactory; 12 | 13 | import util.HBaseHelper; 14 | 15 | // cc ListTablesExample Example listing the existing tables and their descriptors 16 | public class ListTablesExample { 17 | 18 | public static void main(String[] args) throws IOException, InterruptedException { 19 | Configuration conf = HBaseConfiguration.create(); 20 | 21 | HBaseHelper helper = HBaseHelper.getHelper(conf); 22 | helper.dropTable("testtable1"); 23 | helper.dropTable("testtable2"); 24 | helper.dropTable("testtable3"); 25 | helper.createTable("testtable1", "colfam1", "colfam2", "colfam3"); 26 | helper.createTable("testtable2", "colfam1", "colfam2", "colfam3"); 27 | helper.createTable("testtable3", "colfam1", "colfam2", "colfam3"); 28 | 29 | // vv ListTablesExample 30 | Connection connection = ConnectionFactory.createConnection(conf); 31 | Admin admin = connection.getAdmin(); 32 | 33 | HTableDescriptor[] htds = admin.listTables(); 34 | // ^^ ListTablesExample 35 | System.out.println("Printing all tables..."); 36 | // vv ListTablesExample 37 | for (HTableDescriptor htd : htds) { 38 | System.out.println(htd); 39 | } 40 | 41 | HTableDescriptor htd1 = admin.getTableDescriptor( 42 | TableName.valueOf("testtable1")); 43 | // ^^ ListTablesExample 44 | System.out.println("Printing testtable1..."); 45 | // vv ListTablesExample 46 | System.out.println(htd1); 47 | 48 | HTableDescriptor htd2 = admin.getTableDescriptor( 49 | TableName.valueOf("testtable10")); 50 | // ^^ ListTablesExample 51 | System.out.println("Printing testtable10..."); 52 | // vv ListTablesExample 53 | System.out.println(htd2); 54 | // ^^ ListTablesExample 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /ch05/src/main/java/admin/NamespaceDescriptorExample.java: -------------------------------------------------------------------------------- 1 | package admin; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.hbase.NamespaceDescriptor; 6 | 7 | // cc NamespaceDescriptorExample Example how to create a NamespaceDescriptor in code 8 | public class NamespaceDescriptorExample { 9 | 10 | public static void main(String[] args) throws IOException, InterruptedException { 11 | // vv NamespaceDescriptorExample 12 | NamespaceDescriptor.Builder builder = 13 | NamespaceDescriptor.create("testspace"); 14 | builder.addConfiguration("key1", "value1"); 15 | NamespaceDescriptor desc = builder.build(); 16 | System.out.println("Namespace: " + desc); 17 | // ^^ NamespaceDescriptorExample 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /ch05/src/main/java/admin/NamespaceExample.java: -------------------------------------------------------------------------------- 1 | package admin; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.conf.Configuration; 6 | import org.apache.hadoop.hbase.HBaseConfiguration; 7 | import org.apache.hadoop.hbase.NamespaceDescriptor; 8 | import org.apache.hadoop.hbase.TableName; 9 | import org.apache.hadoop.hbase.client.Admin; 10 | import org.apache.hadoop.hbase.client.Connection; 11 | import org.apache.hadoop.hbase.client.ConnectionFactory; 12 | 13 | // cc NamespaceExample Example using the administrative API to create etc. a namespace 14 | public class NamespaceExample { 15 | 16 | public static void main(String[] args) throws IOException, InterruptedException { 17 | // vv NamespaceExample 18 | Configuration conf = HBaseConfiguration.create(); 19 | Connection connection = ConnectionFactory.createConnection(conf); 20 | Admin admin = connection.getAdmin(); 21 | // ^^ NamespaceExample 22 | try { 23 | TableName[] tbls = admin.listTableNamesByNamespace("testspace"); 24 | for (TableName tbl : tbls) { 25 | admin.disableTable(tbl); 26 | admin.deleteTable(tbl); 27 | } 28 | admin.deleteNamespace("testspace"); 29 | } catch (IOException e) { 30 | // ignore 31 | } 32 | // vv NamespaceExample 33 | NamespaceDescriptor namespace = 34 | NamespaceDescriptor.create("testspace").build(); 35 | admin.createNamespace(namespace); 36 | 37 | NamespaceDescriptor namespace2 = 38 | admin.getNamespaceDescriptor("testspace"); 39 | System.out.println("Simple Namespace: " + namespace2); 40 | 41 | NamespaceDescriptor[] list = admin.listNamespaceDescriptors(); 42 | for (NamespaceDescriptor nd : list) { 43 | System.out.println("List Namespace: " + nd); 44 | } 45 | 46 | NamespaceDescriptor namespace3 = 47 | NamespaceDescriptor.create("testspace") 48 | .addConfiguration("Description", "Test Namespace") 49 | .build(); 50 | admin.modifyNamespace(namespace3); 51 | 52 | NamespaceDescriptor namespace4 = 53 | admin.getNamespaceDescriptor("testspace"); 54 | System.out.println("Custom Namespace: " + namespace4); 55 | 56 | admin.deleteNamespace("testspace"); 57 | 58 | NamespaceDescriptor[] list2 = admin.listNamespaceDescriptors(); 59 | for (NamespaceDescriptor nd : list2) { 60 | System.out.println("List Namespace: " + nd); 61 | } 62 | // ^^ NamespaceExample 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /ch05/src/main/java/admin/ServerAndRegionNameExample.java: -------------------------------------------------------------------------------- 1 | package admin; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.conf.Configuration; 6 | import org.apache.hadoop.hbase.HBaseConfiguration; 7 | import org.apache.hadoop.hbase.HColumnDescriptor; 8 | import org.apache.hadoop.hbase.HRegionInfo; 9 | import org.apache.hadoop.hbase.HRegionLocation; 10 | import org.apache.hadoop.hbase.HTableDescriptor; 11 | import org.apache.hadoop.hbase.TableName; 12 | import org.apache.hadoop.hbase.client.Admin; 13 | import org.apache.hadoop.hbase.client.Connection; 14 | import org.apache.hadoop.hbase.client.ConnectionFactory; 15 | import org.apache.hadoop.hbase.client.RegionLocator; 16 | import org.apache.hadoop.hbase.util.Bytes; 17 | 18 | import util.HBaseHelper; 19 | 20 | // cc ServerAndRegionNameExample Shows the use of server and region names 21 | public class ServerAndRegionNameExample { 22 | 23 | public static void main(String[] args) throws IOException, InterruptedException { 24 | Configuration conf = HBaseConfiguration.create(); 25 | HBaseHelper helper = HBaseHelper.getHelper(conf); 26 | helper.dropTable("testtable"); 27 | Connection connection = ConnectionFactory.createConnection(conf); 28 | Admin admin = connection.getAdmin(); 29 | 30 | // vv ServerAndRegionNameExample 31 | TableName tableName = TableName.valueOf("testtable"); 32 | HColumnDescriptor coldef1 = new HColumnDescriptor("colfam1"); 33 | HTableDescriptor desc = new HTableDescriptor(tableName) 34 | .addFamily(coldef1) 35 | .setValue("Description", "Chapter 5 - ServerAndRegionNameExample"); 36 | byte[][] regions = new byte[][] { Bytes.toBytes("ABC"), 37 | Bytes.toBytes("DEF"), Bytes.toBytes("GHI"), Bytes.toBytes("KLM"), 38 | Bytes.toBytes("OPQ"), Bytes.toBytes("TUV") 39 | }; 40 | admin.createTable(desc, regions); 41 | 42 | RegionLocator locator = connection.getRegionLocator(tableName); 43 | HRegionLocation location = locator.getRegionLocation(Bytes.toBytes("Foo")); 44 | HRegionInfo info = location.getRegionInfo(); 45 | System.out.println("Region Name: " + info.getRegionNameAsString()); 46 | System.out.println("Server Name: " + location.getServerName()); 47 | // ^^ ServerAndRegionNameExample 48 | locator.close(); 49 | admin.close(); 50 | connection.close(); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /ch05/src/main/java/admin/TableNameExample.java: -------------------------------------------------------------------------------- 1 | package admin; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.hbase.TableName; 6 | 7 | // cc TableNameExample Example how to create a TableName in code 8 | public class TableNameExample { 9 | 10 | // vv TableNameExample 11 | private static void print(String tablename) { 12 | print(null, tablename); 13 | } 14 | 15 | private static void print(String namespace, String tablename) { 16 | System.out.print("Given Namespace: " + namespace + 17 | ", Tablename: " + tablename + " -> "); 18 | try { 19 | System.out.println(namespace != null ? 20 | TableName.valueOf(namespace, tablename) : 21 | TableName.valueOf(tablename)); 22 | } catch (Exception e) { 23 | System.out.println(e.getClass().getSimpleName() + 24 | ": " + e.getMessage()); 25 | } 26 | } 27 | 28 | public static void main(String[] args) throws IOException, InterruptedException { 29 | print("testtable"); 30 | print("testspace:testtable"); 31 | print("testspace", "testtable"); 32 | print("testspace", "te_st-ta.ble"); 33 | print("", "TestTable-100"); 34 | print("tEsTsPaCe", "te_st-table"); 35 | 36 | print(""); 37 | 38 | // VALID_NAMESPACE_REGEX = "(?:[a-zA-Z_0-9]+)"; 39 | // VALID_TABLE_QUALIFIER_REGEX = "(?:[a-zA-Z_0-9][a-zA-Z_0-9-.]*)"; 40 | print(".testtable"); 41 | print("te_st-space", "te_st-table"); 42 | print("tEsTsPaCe", "te_st-table@dev"); 43 | } 44 | // ^^ TableNameExample 45 | } 46 | -------------------------------------------------------------------------------- /ch05/src/main/java/coprocessor/DelayRegionCloseObserver.java: -------------------------------------------------------------------------------- 1 | package coprocessor; 2 | 3 | import java.io.IOException; 4 | import java.util.Random; 5 | 6 | import org.apache.commons.logging.Log; 7 | import org.apache.commons.logging.LogFactory; 8 | 9 | import org.apache.hadoop.fs.FileSystem; 10 | import org.apache.hadoop.fs.Path; 11 | import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver; 12 | import org.apache.hadoop.hbase.coprocessor.ObserverContext; 13 | import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; 14 | import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper; 15 | import org.apache.hadoop.hbase.io.Reference; 16 | import org.apache.hadoop.hbase.io.hfile.CacheConfig; 17 | import org.apache.hadoop.hbase.regionserver.HRegion; 18 | import org.apache.hadoop.hbase.regionserver.StoreFile; 19 | 20 | // cc DelayRegionCloseObserver Special test observer creating delays 21 | public class DelayRegionCloseObserver extends BaseRegionObserver { 22 | public static final Log LOG = LogFactory.getLog(HRegion.class); 23 | 24 | // vv DelayRegionCloseObserver 25 | private Random rnd = new Random(); 26 | 27 | @Override 28 | public StoreFile.Reader preStoreFileReaderOpen( 29 | ObserverContext ctx, FileSystem fs, Path p, 30 | FSDataInputStreamWrapper in, long size, CacheConfig cacheConf, Reference r, 31 | StoreFile.Reader reader) throws IOException { 32 | try { 33 | long delay = rnd.nextInt(3); 34 | LOG.info("@@@ Delaying region " + 35 | ctx.getEnvironment().getRegion().getRegionInfo(). 36 | getRegionNameAsString() + " for " + delay + " seconds..."); 37 | Thread.sleep(delay * 1000); 38 | } catch (InterruptedException ie) { 39 | LOG.error(ie); 40 | } 41 | return reader; 42 | } 43 | // ^^ DelayRegionCloseObserver 44 | } 45 | -------------------------------------------------------------------------------- /ch05/src/main/java/coprocessor/DelayingMasterObserver.java: -------------------------------------------------------------------------------- 1 | package coprocessor; 2 | 3 | import java.io.IOException; 4 | import java.util.Random; 5 | 6 | import org.apache.commons.logging.Log; 7 | import org.apache.commons.logging.LogFactory; 8 | import org.apache.hadoop.hbase.CoprocessorEnvironment; 9 | import org.apache.hadoop.hbase.HRegionInfo; 10 | import org.apache.hadoop.hbase.ServerName; 11 | import org.apache.hadoop.hbase.coprocessor.BaseMasterObserver; 12 | import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment; 13 | import org.apache.hadoop.hbase.master.AssignmentListener; 14 | import org.apache.hadoop.hbase.regionserver.HRegion; 15 | 16 | // cc DelayingMasterObserver Special master observer that delays region asignments 17 | // vv DelayingMasterObserver 18 | public class DelayingMasterObserver extends BaseMasterObserver 19 | implements AssignmentListener { 20 | public static final Log LOG = LogFactory.getLog(HRegion.class); 21 | 22 | private Random rnd = new Random(); 23 | 24 | @Override 25 | public void regionOpened(HRegionInfo hRegionInfo, ServerName serverName) { 26 | try { 27 | if (hRegionInfo.getTable().getQualifierAsString().equals("testtable")) { 28 | long delay = rnd.nextInt(3); 29 | LOG.info("@@@ Delaying region " + 30 | hRegionInfo.getRegionNameAsString() + 31 | " for " + delay + " seconds..."); 32 | Thread.sleep(delay * 1000); 33 | } 34 | } catch (InterruptedException ie) { 35 | LOG.error(ie); 36 | } 37 | } 38 | 39 | @Override 40 | public void regionClosed(HRegionInfo hRegionInfo) { 41 | 42 | } 43 | 44 | @Override 45 | public void start(CoprocessorEnvironment ctx) throws IOException { 46 | MasterCoprocessorEnvironment env = (MasterCoprocessorEnvironment) ctx; 47 | env.getMasterServices().getAssignmentManager().registerListener(this); 48 | } 49 | } 50 | // ^^ DelayingMasterObserver 51 | -------------------------------------------------------------------------------- /ch05/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO, stdout 2 | 3 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 4 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 5 | 6 | # Pattern to output the caller's file name and line number. 7 | log4j.appender.stdout.layout.ConversionPattern=%5p [%t] (%F:%L) - %m%n 8 | 9 | log4j.logger.org.apache=WARN -------------------------------------------------------------------------------- /ch06/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDeleteType.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Autogenerated by Thrift Compiler (0.9.2) 3 | * 4 | * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 5 | * @generated 6 | */ 7 | package org.apache.hadoop.hbase.thrift2.generated; 8 | 9 | 10 | import java.util.Map; 11 | import java.util.HashMap; 12 | import org.apache.thrift.TEnum; 13 | 14 | /** 15 | * Specify type of delete: 16 | * - DELETE_COLUMN means exactly one version will be removed, 17 | * - DELETE_COLUMNS means previous versions will also be removed. 18 | */ 19 | public enum TDeleteType implements org.apache.thrift.TEnum { 20 | DELETE_COLUMN(0), 21 | DELETE_COLUMNS(1); 22 | 23 | private final int value; 24 | 25 | private TDeleteType(int value) { 26 | this.value = value; 27 | } 28 | 29 | /** 30 | * Get the integer value of this enum value, as defined in the Thrift IDL. 31 | */ 32 | public int getValue() { 33 | return value; 34 | } 35 | 36 | /** 37 | * Find a the enum type by its integer value, as defined in the Thrift IDL. 38 | * @return null if the value is not found. 39 | */ 40 | public static TDeleteType findByValue(int value) { 41 | switch (value) { 42 | case 0: 43 | return DELETE_COLUMN; 44 | case 1: 45 | return DELETE_COLUMNS; 46 | default: 47 | return null; 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /ch06/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDurability.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Autogenerated by Thrift Compiler (0.9.2) 3 | * 4 | * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 5 | * @generated 6 | */ 7 | package org.apache.hadoop.hbase.thrift2.generated; 8 | 9 | 10 | import java.util.Map; 11 | import java.util.HashMap; 12 | import org.apache.thrift.TEnum; 13 | 14 | /** 15 | * Specify Durability: 16 | * - SKIP_WAL means do not write the Mutation to the WAL. 17 | * - ASYNC_WAL means write the Mutation to the WAL asynchronously, 18 | * - SYNC_WAL means write the Mutation to the WAL synchronously, 19 | * - FSYNC_WAL means Write the Mutation to the WAL synchronously and force the entries to disk. 20 | */ 21 | public enum TDurability implements org.apache.thrift.TEnum { 22 | SKIP_WAL(1), 23 | ASYNC_WAL(2), 24 | SYNC_WAL(3), 25 | FSYNC_WAL(4); 26 | 27 | private final int value; 28 | 29 | private TDurability(int value) { 30 | this.value = value; 31 | } 32 | 33 | /** 34 | * Get the integer value of this enum value, as defined in the Thrift IDL. 35 | */ 36 | public int getValue() { 37 | return value; 38 | } 39 | 40 | /** 41 | * Find a the enum type by its integer value, as defined in the Thrift IDL. 42 | * @return null if the value is not found. 43 | */ 44 | public static TDurability findByValue(int value) { 45 | switch (value) { 46 | case 1: 47 | return SKIP_WAL; 48 | case 2: 49 | return ASYNC_WAL; 50 | case 3: 51 | return SYNC_WAL; 52 | case 4: 53 | return FSYNC_WAL; 54 | default: 55 | return null; 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /ch06/src/main/java/pig/JsonStringToMap.java: -------------------------------------------------------------------------------- 1 | package pig; 2 | 3 | import java.io.IOException; 4 | import java.util.Map; 5 | 6 | import org.apache.pig.EvalFunc; 7 | import org.apache.pig.backend.executionengine.ExecException; 8 | import org.apache.pig.data.Tuple; 9 | import org.json.simple.JSONObject; 10 | import org.json.simple.parser.JSONParser; 11 | import org.json.simple.parser.ParseException; 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | 15 | import com.google.common.collect.Maps; 16 | 17 | /** 18 | *

Transforms a Json string into a Pig map.
19 | * Only goes 1 level deep -- all value representations are their toString() 20 | * representations.

21 | * 22 | * From Elephant Bird: https://github.com/kevinweil/elephant-bird/blob/master/ 23 | * src/java/com/twitter/elephantbird/pig/piggybank/JsonStringToMap.java 24 | */ 25 | @SuppressWarnings("rawtypes") 26 | public class JsonStringToMap extends EvalFunc { 27 | private static final Logger LOG = LoggerFactory.getLogger( 28 | JsonStringToMap.class); 29 | private final JSONParser jsonParser = new JSONParser(); 30 | 31 | @Override 32 | public Map exec(Tuple input) throws IOException { 33 | try { 34 | if (input == null || input.size() < 1) { 35 | throw new IOException("Not enough arguments to " + 36 | this.getClass().getName() + ": got " + input.size() + 37 | ", expected at least 1"); 38 | } 39 | 40 | if (input.get(0) == null) { 41 | return null; 42 | } 43 | 44 | String jsonLiteral = (String) input.get(0); 45 | return parseStringToMap(jsonLiteral); 46 | } catch (ExecException e) { 47 | LOG.warn("Error in " + getClass() + " with input " + input, e); 48 | throw new IOException(e); 49 | } 50 | } 51 | 52 | protected Map parseStringToMap(String line) { 53 | try { 54 | Map values = Maps.newHashMap(); 55 | JSONObject jsonObj = (JSONObject) jsonParser.parse(line); 56 | for (Object key : jsonObj.keySet()) { 57 | Object value = jsonObj.get(key); 58 | values.put(key.toString(), value != null ? value.toString() : null); 59 | } 60 | return values; 61 | } catch (ParseException e) { 62 | LOG.warn("Could not json-decode string: " + line, e); 63 | return null; 64 | } catch (NumberFormatException e) { 65 | LOG.warn("Very big number exceeds the scale of long: " + line, e); 66 | return null; 67 | } 68 | } 69 | 70 | } 71 | -------------------------------------------------------------------------------- /ch06/src/main/java/rest/RestExample.java: -------------------------------------------------------------------------------- 1 | package rest; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.conf.Configuration; 6 | import org.apache.hadoop.hbase.HBaseConfiguration; 7 | import org.apache.hadoop.hbase.client.Get; 8 | import org.apache.hadoop.hbase.client.Result; 9 | import org.apache.hadoop.hbase.client.ResultScanner; 10 | import org.apache.hadoop.hbase.client.Scan; 11 | import org.apache.hadoop.hbase.rest.client.Client; 12 | import org.apache.hadoop.hbase.rest.client.Cluster; 13 | import org.apache.hadoop.hbase.rest.client.RemoteHTable; 14 | import org.apache.hadoop.hbase.util.Bytes; 15 | 16 | import util.HBaseHelper; 17 | 18 | // cc RestExample Example of using the REST client classes 19 | public class RestExample { 20 | 21 | public static void main(String[] args) throws IOException { 22 | Configuration conf = HBaseConfiguration.create(); 23 | 24 | HBaseHelper helper = HBaseHelper.getHelper(conf); 25 | helper.dropTable("testtable"); 26 | helper.createTable("testtable", "colfam1"); 27 | System.out.println("Adding rows to table..."); 28 | helper.fillTable("testtable", 1, 100, 10, "colfam1"); 29 | 30 | // vv RestExample 31 | Cluster cluster = new Cluster(); 32 | cluster.add("localhost", 8080); // co RestExample-1-Cluster Set up a cluster list adding all known REST server hosts. 33 | 34 | Client client = new Client(cluster); // co RestExample-2-Client Create the client handling the HTTP communication. 35 | 36 | RemoteHTable table = new RemoteHTable(client, "testtable"); // co RestExample-3-Table Create a remote table instance, wrapping the REST access into a familiar interface. 37 | 38 | Get get = new Get(Bytes.toBytes("row-30")); // co RestExample-4-Get Perform a get operation as if it were a direct HBase connection. 39 | get.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("col-3")); 40 | Result result1 = table.get(get); 41 | 42 | System.out.println("Get result1: " + result1); 43 | 44 | Scan scan = new Scan(); 45 | scan.setStartRow(Bytes.toBytes("row-10")); 46 | scan.setStopRow(Bytes.toBytes("row-15")); 47 | scan.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("col-5")); 48 | ResultScanner scanner = table.getScanner(scan); // co RestExample-5-Scan Scan the table, again, the same approach as if using the native Java API. 49 | 50 | for (Result result2 : scanner) { 51 | System.out.println("Scan row[" + Bytes.toString(result2.getRow()) + 52 | "]: " + result2); 53 | } 54 | // ^^ RestExample 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /ch07/src/main/assembly/job.xml: -------------------------------------------------------------------------------- 1 | 5 | job 6 | 7 | jar 8 | 9 | false 10 | 11 | 12 | false 13 | runtime 14 | lib 15 | 16 | 17 | false 18 | system 19 | lib 20 | 21 | ${artifact.groupId}:${artifact.artifactId} 22 | 23 | 24 | 25 | 26 | 27 | ${basedir}/target/classes 28 | / 29 | 30 | *.jar 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /ch07/src/main/java/mapreduce/BytesSplit.java: -------------------------------------------------------------------------------- 1 | package mapreduce; 2 | 3 | import org.apache.hadoop.hbase.util.Bytes; 4 | 5 | // cc BytesSplit Example for splitting a key range into pieces 6 | public class BytesSplit { 7 | 8 | public static void main(String[] args) { 9 | // vv BytesSplit 10 | byte[][] splits = Bytes.split(Bytes.toBytes(0), Bytes.toBytes(100), 9); // co BytesSplit-1-9Splits The number defines the amount of splits performed. Splitting one region nine times results in ten parts. 11 | int n = 0; 12 | for (byte[] split : splits) { 13 | System.out.println("Split key[" + ++n + "]: " + 14 | Bytes.toInt(split)); 15 | } 16 | // ^^ BytesSplit 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /ch07/src/main/java/mapreduce/Driver.java: -------------------------------------------------------------------------------- 1 | package mapreduce; 2 | 3 | import org.apache.hadoop.util.ProgramDriver; 4 | 5 | /** 6 | * Offers choices for included MapReduce jobs. 7 | */ 8 | public class Driver { 9 | 10 | /** 11 | * Main entry point for jar file. 12 | * 13 | * @param args The command line parameters. 14 | * @throws Throwable When the selection fails. 15 | */ 16 | public static void main(String[] args) throws Throwable { 17 | ProgramDriver pgd = new ProgramDriver(); 18 | pgd.addClass(ImportFromFile.NAME, ImportFromFile.class, 19 | "Import from file"); 20 | pgd.addClass(ImportFromFile2.NAME, ImportFromFile2.class, 21 | "Import from file (with dependencies)"); 22 | pgd.addClass(AnalyzeData.NAME, AnalyzeData.class, 23 | "Analyze imported JSON"); 24 | pgd.addClass(AnalyzeSnapshotData.NAME, AnalyzeSnapshotData.class, 25 | "Analyze imported JSON from snapshot"); 26 | pgd.addClass(ParseJson.NAME, ParseJson.class, 27 | "Parse JSON into columns"); 28 | pgd.addClass(ParseJson2.NAME, ParseJson2.class, 29 | "Parse JSON into columns (map only)"); 30 | pgd.addClass(ParseJsonMulti.NAME, ParseJsonMulti.class, 31 | "Parse JSON into multiple tables"); 32 | pgd.driver(args); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /ch07/src/main/java/mapreduce/InvalidReducerOverride.java: -------------------------------------------------------------------------------- 1 | package mapreduce; 2 | 3 | import org.apache.hadoop.io.Text; 4 | import org.apache.hadoop.io.Writable; 5 | import org.apache.hadoop.mapreduce.Reducer; 6 | 7 | import java.io.IOException; 8 | import java.util.Iterator; 9 | 10 | public class InvalidReducerOverride { 11 | 12 | static class InvalidOverrideReduce 13 | extends Reducer { 14 | //@Override 15 | protected void reduce(Writable key, Iterator values, Context context) 16 | throws IOException, InterruptedException { 17 | context.write(key, new Text()); 18 | } 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /ch08/bin/doprotoc.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Runs the Google Protocol Buffer compiler. 4 | # 5 | # Requirements: protoc 6 | # 7 | # Note: 8 | # - For MacOS with Homebrew run: $ brew install protobuf 9 | # 10 | set -e 11 | 12 | PROTO_BASE_DIR="src/main/protobuf" 13 | 14 | PROTO_SCANCONTROL="$PROTO_BASE_DIR/ScanControlService.proto" 15 | 16 | PROTOS="$PROTO_SCANCONTROL" 17 | 18 | PATH_GENERATED_BASE="src/main/java" 19 | PATH_GENERATED_COPROS="$PATH_GENERATED_BASE/coprocessor/generated" 20 | 21 | PATHS="$PATH_GENERATED_COPROS" 22 | 23 | # check all is well 24 | if [ ! -d "$PROTO_BASE_DIR" ]; then 25 | echo "Error: this script must run in the project root directort... exiting!" 26 | exit -1 27 | fi 28 | 29 | # check if output directory exists, if not create it 30 | for loc in $PATHS; do 31 | if [ ! -d "$loc" ]; then 32 | echo "creating directory: $loc" 33 | mkdir "$loc" 34 | fi 35 | done 36 | 37 | # run protocol buffer compiler 38 | for proto in $PROTOS; do 39 | if [ -f "$proto" ]; then 40 | echo "compiling protocol: $proto" 41 | protoc -I$PROTO_BASE_DIR --java_out=$PATH_GENERATED_BASE $proto 42 | fi 43 | done 44 | 45 | echo "done." 46 | -------------------------------------------------------------------------------- /ch08/bin/run.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | # 3 | # run.sh: helps executing the examples by setting up the Java CLASSPATH. 4 | # 5 | 6 | # get the current directory 7 | bin=`dirname "$0"` 8 | bin=`cd "$bin">/dev/null; pwd` 9 | 10 | # check command line args 11 | if [[ $# == 0 ]]; then 12 | echo "usage: $(basename $0) " 13 | exit 1; 14 | fi 15 | 16 | # set up Maven environment 17 | MVN="mvn" 18 | if [ "$MAVEN_HOME" != "" ]; then 19 | MVN=${MAVEN_HOME}/bin/mvn 20 | fi 21 | 22 | # classpath initially contains $HBASE_CONF_DIR 23 | CLASSPATH="${HBASE_CONF_DIR}" 24 | 25 | # add classes first, triggers log4j.properties priority 26 | if [ -d "${bin}/../target/classes" ]; then 27 | CLASSPATH=${CLASSPATH}:${bin}/../target/classes 28 | fi 29 | 30 | # create and cache Maven classpath 31 | cpfile="${bin}/../target/cached_classpath.txt" 32 | if [ ! -f "${cpfile}" ]; then 33 | ${MVN} -f "${bin}/../pom.xml" dependency:build-classpath -Dmdep.outputFile="${cpfile}" &> /dev/null 34 | fi 35 | CLASSPATH=${CLASSPATH}:`cat "${cpfile}"` 36 | 37 | JAVA=$JAVA_HOME/bin/java 38 | JAVA_HEAP_MAX=-Xmx512m 39 | 40 | "$JAVA" $JAVA_HEAP_MAX -classpath "$CLASSPATH" "$@" -------------------------------------------------------------------------------- /ch08/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | 6 | com.larsgeorge 7 | hbase-book 8 | 2.0 9 | ../pom.xml 10 | 11 | 12 | com.larsgeorge 13 | hbase-book-ch08 14 | 2.0 15 | jar 16 | 17 | HBase Book Chapter 8 18 | 19 | 20 | 21 | com.larsgeorge 22 | hbase-book-common 23 | ${project.version} 24 | 25 | 26 | -------------------------------------------------------------------------------- /ch08/src/main/protobuf/ScanControlService.proto: -------------------------------------------------------------------------------- 1 | option java_package = "coprocessor.generated"; 2 | option java_outer_classname = "ScanControlProtos"; 3 | option java_generic_services = true; 4 | option java_generate_equals_and_hash = true; 5 | option optimize_for = SPEED; 6 | 7 | message ScanControlRequest { 8 | } 9 | 10 | message ScanControlResponse { 11 | } 12 | 13 | service ScanControlService { 14 | rpc resumeScan(ScanControlRequest) 15 | returns (ScanControlResponse); 16 | } 17 | 18 | -------------------------------------------------------------------------------- /ch09/bin/run.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | # 3 | # run.sh: helps executing the examples by setting up the Java CLASSPATH. 4 | # 5 | 6 | # get the current directory 7 | bin=`dirname "$0"` 8 | bin=`cd "$bin">/dev/null; pwd` 9 | 10 | # check command line args 11 | if [[ $# == 0 ]]; then 12 | echo "usage: $(basename $0) " 13 | exit 1; 14 | fi 15 | 16 | # set up Maven environment 17 | MVN="mvn" 18 | if [ "$MAVEN_HOME" != "" ]; then 19 | MVN=${MAVEN_HOME}/bin/mvn 20 | fi 21 | 22 | # classpath initially contains $HBASE_CONF_DIR 23 | CLASSPATH="${HBASE_CONF_DIR}" 24 | 25 | # add classes first, triggers log4j.properties priority 26 | if [ -d "${bin}/../target/classes" ]; then 27 | CLASSPATH=${CLASSPATH}:${bin}/../target/classes 28 | fi 29 | 30 | # create and cache Maven classpath 31 | cpfile="${bin}/../target/cached_classpath.txt" 32 | if [ ! -f "${cpfile}" ]; then 33 | ${MVN} -f "${bin}/../pom.xml" dependency:build-classpath -Dmdep.outputFile="${cpfile}" &> /dev/null 34 | fi 35 | CLASSPATH=${CLASSPATH}:`cat "${cpfile}"` 36 | 37 | JAVA=$JAVA_HOME/bin/java 38 | JAVA_HEAP_MAX=-Xmx512m 39 | 40 | "$JAVA" $JAVA_HEAP_MAX -classpath "$CLASSPATH" "$@" -------------------------------------------------------------------------------- /ch09/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | 6 | com.larsgeorge 7 | hbase-book 8 | 2.0 9 | ../pom.xml 10 | 11 | 12 | com.larsgeorge 13 | hbase-book-ch09 14 | 2.0 15 | jar 16 | 17 | HBase Book Chapter 9 18 | 19 | 20 | 21 | com.larsgeorge 22 | hbase-book-common 23 | ${project.version} 24 | 25 | 26 | -------------------------------------------------------------------------------- /ch09/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO, stdout 2 | 3 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 4 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 5 | 6 | # Pattern to output the caller's file name and line number. 7 | log4j.appender.stdout.layout.ConversionPattern=%5p [%t] (%F:%L) - %m%n 8 | 9 | log4j.logger.org.apache=WARN -------------------------------------------------------------------------------- /ch11/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | com.larsgeorge 6 | hbase-book 7 | 2.0 8 | ../pom.xml 9 | 10 | 11 | com.larsgeorge 12 | hbase-book-ch11 13 | 2.0 14 | jar 15 | 16 | HBase Book Chapter 11 17 | 18 | 19 | 20 | com.larsgeorge 21 | hbase-book-common 22 | ${project.version} 23 | 24 | 25 | org.apache.hadoop 26 | hadoop-mapreduce-client-core 27 | 2.6.0 28 | provided 29 | 30 | 31 | hadoop-yarn-common 32 | org.apache.hadoop 33 | 34 | 35 | avro 36 | org.apache.avro 37 | 38 | 39 | slf4j-log4j12 40 | org.slf4j 41 | 42 | 43 | guice-servlet 44 | com.google.inject.extensions 45 | 46 | 47 | netty 48 | io.netty 49 | 50 | 51 | 52 | 53 | com.googlecode.json-simple 54 | json-simple 55 | 1.1.1 56 | 57 | 58 | junit 59 | junit 60 | 61 | 62 | 63 | 64 | 65 | -------------------------------------------------------------------------------- /ch12/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | com.larsgeorge 6 | hbase-book 7 | 2.0 8 | ../pom.xml 9 | 10 | 11 | com.larsgeorge 12 | hbase-book-ch12 13 | 2.0 14 | jar 15 | 16 | HBase Book Chapter 12 17 | 18 | 19 | 20 | com.larsgeorge 21 | hbase-book-common 22 | ${project.version} 23 | 24 | 25 | org.apache.htrace 26 | htrace-zipkin 27 | 3.1.0-incubating 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /ch12/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO, stdout 2 | 3 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 4 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 5 | 6 | # Pattern to output the caller's file name and line number. 7 | log4j.appender.stdout.layout.ConversionPattern=%5p [%t] (%F:%L) - %m%n 8 | 9 | log4j.logger.org.apache=WARN 10 | log4j.logger.org.apache.zookeeper.ZooKeeper=WARN -------------------------------------------------------------------------------- /ch13/bin/run.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | # 3 | # run.sh: helps executing the examples by setting up the Java CLASSPATH. 4 | # 5 | 6 | # get the current directory 7 | bin=`dirname "$0"` 8 | bin=`cd "$bin">/dev/null; pwd` 9 | 10 | # check command line args 11 | if [[ $# == 0 ]]; then 12 | echo "usage: $(basename $0) " 13 | exit 1; 14 | fi 15 | 16 | # set up Maven environment 17 | MVN="mvn" 18 | if [ "$MAVEN_HOME" != "" ]; then 19 | MVN=${MAVEN_HOME}/bin/mvn 20 | fi 21 | 22 | # classpath initially contains $HBASE_CONF_DIR 23 | CLASSPATH="${HBASE_CONF_DIR}" 24 | 25 | # add classes first, triggers log4j.properties priority 26 | if [ -d "${bin}/../target/classes" ]; then 27 | CLASSPATH=${CLASSPATH}:${bin}/../target/classes 28 | fi 29 | 30 | # create and cache Maven classpath 31 | cpfile="${bin}/../target/cached_classpath.txt" 32 | if [ ! -f "${cpfile}" ]; then 33 | ${MVN} -f "${bin}/../pom.xml" dependency:build-classpath -Dmdep.outputFile="${cpfile}" &> /dev/null 34 | fi 35 | CLASSPATH=${CLASSPATH}:`cat "${cpfile}"`:${HTD_CH13_CLASSPATH} 36 | 37 | JAVA=$JAVA_HOME/bin/java 38 | JAVA_HEAP_MAX=-Xmx512m 39 | 40 | "$JAVA" $JAVA_HEAP_MAX -classpath "$CLASSPATH" "$@" -------------------------------------------------------------------------------- /ch13/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | com.larsgeorge 6 | hbase-book 7 | 2.0 8 | ../pom.xml 9 | 10 | 11 | com.larsgeorge 12 | hbase-book-ch13 13 | 2.0 14 | jar 15 | 16 | HBase Book Chapter 13 17 | 18 | 19 | 20 | com.larsgeorge 21 | hbase-book-common 22 | ${project.version} 23 | 24 | 25 | org.apache.thrift 26 | libthrift 27 | 0.9.2 28 | 29 | 30 | slf4j-api 31 | org.slf4j 32 | 33 | 34 | slf4j-log4j12 35 | org.slf4j 36 | 37 | 38 | junit 39 | junit 40 | 41 | 42 | httpclient 43 | org.apache.httpcomponents 44 | 45 | 46 | commons-lang 47 | commons-lang 48 | 49 | 50 | servlet-api 51 | javax.servlet 52 | 53 | 54 | httpcore 55 | org.apache.httpcomponents 56 | 57 | 58 | 59 | 60 | 61 | commons-io 62 | commons-io 63 | 2.4 64 | 65 | 66 | log4j 67 | log4j 68 | 1.2.17 69 | 70 | 71 | 72 | -------------------------------------------------------------------------------- /ch13/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDeleteType.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Autogenerated by Thrift Compiler (0.9.2) 3 | * 4 | * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 5 | * @generated 6 | */ 7 | package org.apache.hadoop.hbase.thrift2.generated; 8 | 9 | 10 | import java.util.Map; 11 | import java.util.HashMap; 12 | import org.apache.thrift.TEnum; 13 | 14 | /** 15 | * Specify type of delete: 16 | * - DELETE_COLUMN means exactly one version will be removed, 17 | * - DELETE_COLUMNS means previous versions will also be removed. 18 | */ 19 | public enum TDeleteType implements org.apache.thrift.TEnum { 20 | DELETE_COLUMN(0), 21 | DELETE_COLUMNS(1); 22 | 23 | private final int value; 24 | 25 | private TDeleteType(int value) { 26 | this.value = value; 27 | } 28 | 29 | /** 30 | * Get the integer value of this enum value, as defined in the Thrift IDL. 31 | */ 32 | public int getValue() { 33 | return value; 34 | } 35 | 36 | /** 37 | * Find a the enum type by its integer value, as defined in the Thrift IDL. 38 | * @return null if the value is not found. 39 | */ 40 | public static TDeleteType findByValue(int value) { 41 | switch (value) { 42 | case 0: 43 | return DELETE_COLUMN; 44 | case 1: 45 | return DELETE_COLUMNS; 46 | default: 47 | return null; 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /ch13/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDurability.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Autogenerated by Thrift Compiler (0.9.2) 3 | * 4 | * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 5 | * @generated 6 | */ 7 | package org.apache.hadoop.hbase.thrift2.generated; 8 | 9 | 10 | import java.util.Map; 11 | import java.util.HashMap; 12 | import org.apache.thrift.TEnum; 13 | 14 | /** 15 | * Specify Durability: 16 | * - SKIP_WAL means do not write the Mutation to the WAL. 17 | * - ASYNC_WAL means write the Mutation to the WAL asynchronously, 18 | * - SYNC_WAL means write the Mutation to the WAL synchronously, 19 | * - FSYNC_WAL means Write the Mutation to the WAL synchronously and force the entries to disk. 20 | */ 21 | public enum TDurability implements org.apache.thrift.TEnum { 22 | SKIP_WAL(1), 23 | ASYNC_WAL(2), 24 | SYNC_WAL(3), 25 | FSYNC_WAL(4); 26 | 27 | private final int value; 28 | 29 | private TDurability(int value) { 30 | this.value = value; 31 | } 32 | 33 | /** 34 | * Get the integer value of this enum value, as defined in the Thrift IDL. 35 | */ 36 | public int getValue() { 37 | return value; 38 | } 39 | 40 | /** 41 | * Find a the enum type by its integer value, as defined in the Thrift IDL. 42 | * @return null if the value is not found. 43 | */ 44 | public static TDurability findByValue(int value) { 45 | switch (value) { 46 | case 1: 47 | return SKIP_WAL; 48 | case 2: 49 | return ASYNC_WAL; 50 | case 3: 51 | return SYNC_WAL; 52 | case 4: 53 | return FSYNC_WAL; 54 | default: 55 | return null; 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /cluster/Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | CLUSTER_DOMAIN = "hbase.book" 5 | 6 | Vagrant.configure("2") do |config| 7 | config.vm.box = "centos/7" 8 | 9 | config.vm.define "master-1" do |node| 10 | node.vm.hostname = "master-1.#{CLUSTER_DOMAIN}" 11 | node.vm.network :private_network, ip: "10.10.0.10" 12 | node.vm.provider "virtualbox" do |v| 13 | v.name = "HBase Book - Master 1" 14 | end 15 | end 16 | 17 | config.vm.define "master-2" do |node| 18 | node.vm.hostname = "master-2.#{CLUSTER_DOMAIN}" 19 | node.vm.network :private_network, ip: "10.10.0.11" 20 | node.vm.provider "virtualbox" do |v| 21 | v.name = "HBase Book - Master 2" 22 | end 23 | end 24 | 25 | config.vm.define "worker-1" do |node| 26 | node.vm.hostname = "worker-1.#{CLUSTER_DOMAIN}" 27 | node.vm.network :private_network, ip: "10.10.0.20" 28 | node.vm.provider "virtualbox" do |v| 29 | v.name = "HBase Book - Worker 1" 30 | end 31 | end 32 | 33 | config.vm.define "worker-2" do |node| 34 | node.vm.hostname = "worker-2.#{CLUSTER_DOMAIN}" 35 | node.vm.network :private_network, ip: "10.10.0.21" 36 | node.vm.provider "virtualbox" do |v| 37 | v.name = "HBase Book - Worker 2" 38 | end 39 | end 40 | 41 | config.vm.define "worker-3" do |node| 42 | node.vm.hostname = "worker-3.#{CLUSTER_DOMAIN}" 43 | node.vm.network :private_network, ip: "10.10.0.22" 44 | node.vm.provider "virtualbox" do |v| 45 | v.name = "HBase Book - Worker 3" 46 | end 47 | end 48 | end -------------------------------------------------------------------------------- /cluster/ansible/README.md: -------------------------------------------------------------------------------- 1 | # HBase Cluster Setup 2 | 3 | This repository contains the necessary (Ansible)[https://www.ansible.com/] playbooks to set up the test cluster used while writing the "HBase - The Definitive Guide" (2. Edition) book. 4 | 5 | The use is as such: there is an *install* playbook, and a *services* playbook. The former does an idempotent setup of the software itself and configures the services, with or without security. The latter allows to start and stop the services and is implicitly invoked by the former. Both are explained next. 6 | 7 | ## Installation Playbook 8 | 9 | Ansible is configuration management system that executes playbooks, others call them recipes, on remote hosts. A common concept with these systems is that they work best in practice when the contained steps are repeatable, or otherwise called idempotent. This allows for an administrator to repeatedly execute the playbook (for example, after something was added or changed) and end up with a working cluster. Ansible supports the notion of skipping steps that are not necessary, which is used for the installation playbook where possible. 10 | 11 | The playbook is executed like this, in its most basic form: 12 | 13 | ``` 14 | $ ansible-playbook install.yml -b -u hbasebook -i inventories/hbasebook.inv 15 | ``` 16 | 17 | Note that `hbasebook` is the technical user account used to set up the cluster. 18 | 19 | You can customize what you want to set up by adding tags to the command, for example: 20 | 21 | ``` 22 | $ ansible-playbook services.yml -b -u hbasebook -i inventories/hbasebook.inv --tags "hbase" 23 | ``` 24 | 25 | This would only apply the steps needed for HBase. 26 | 27 | ## Services Playbook 28 | 29 | The services playbook start and/or stops the various services. Called without further options it restarts all services: 30 | 31 | ``` 32 | $ ansible-playbook services.yml -b -u hbasebook -i inventories/hbasebook.inv 33 | ``` 34 | 35 | Like with the above you can customize the call by adding tags, such as `start` to start all services. To just restart HBase do this: 36 | 37 | ``` 38 | $ ansible-playbook services.yml -b -u hbasebook -i inventories/hbasebook.inv --tags "stop-hbase,start-hbase" 39 | ``` 40 | -------------------------------------------------------------------------------- /cluster/ansible/group_vars/hbase_backup_masters: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | HBASE_MODE: "backup" -------------------------------------------------------------------------------- /cluster/ansible/group_vars/hbase_master: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | HBASE_MODE: "master" -------------------------------------------------------------------------------- /cluster/ansible/group_vars/hbase_workers: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | HBASE_MODE: "worker" -------------------------------------------------------------------------------- /cluster/ansible/group_vars/kdc-server: -------------------------------------------------------------------------------- 1 | --- 2 | -------------------------------------------------------------------------------- /cluster/ansible/group_vars/zookeepers: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | zookeeper: true -------------------------------------------------------------------------------- /cluster/ansible/host_vars/master-1: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | ZOOKEEPER_ID: 1 4 | 5 | namenode: true -------------------------------------------------------------------------------- /cluster/ansible/host_vars/master-2: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | ZOOKEEPER_ID: 2 -------------------------------------------------------------------------------- /cluster/ansible/host_vars/master-3: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | ZOOKEEPER_ID: 3 -------------------------------------------------------------------------------- /cluster/ansible/inventories/cluster.inv: -------------------------------------------------------------------------------- 1 | master-1 ansible_ssh_host=10.10.0.10 ansible_ssh_user='vagrant' ansible_ssh_private_key_file='.vagrant/machines/master-1/virtualbox/private_key' namenode=true 2 | master-2 ansible_ssh_host=10.10.0.11 ansible_ssh_user='vagrant' ansible_ssh_private_key_file='.vagrant/machines/master-2/virtualbox/private_key' 3 | worker-1 ansible_ssh_host=10.10.0.20 ansible_ssh_user='vagrant' ansible_ssh_private_key_file='.vagrant/machines/worker-1/virtualbox/private_key' 4 | worker-2 ansible_ssh_host=10.10.0.21 ansible_ssh_user='vagrant' ansible_ssh_private_key_file='.vagrant/machines/worker-2/virtualbox/private_key' 5 | worker-3 ansible_ssh_host=10.10.0.22 ansible_ssh_user='vagrant' ansible_ssh_private_key_file='.vagrant/machines/worker-3/virtualbox/private_key' 6 | 7 | [masters] 8 | master-[1:2] 9 | 10 | [workers] 11 | worker-[1:3] 12 | 13 | [hbase_master] 14 | master-1 15 | 16 | [kdc_server] 17 | master-1 18 | 19 | [hbase_backup_masters] 20 | master-[2:2] 21 | 22 | [cluster:children] 23 | masters 24 | workers 25 | 26 | [hbase_workers:children] 27 | workers 28 | 29 | [zookeepers:children] 30 | masters 31 | 32 | [all:children] 33 | cluster 34 | -------------------------------------------------------------------------------- /cluster/ansible/inventories/hbasebook.inv: -------------------------------------------------------------------------------- 1 | master-1 ansible_host=10.0.20.1 2 | master-2 ansible_host=10.0.20.2 3 | master-3 ansible_host=10.0.20.3 4 | worker-1 ansible_host=10.0.20.10 5 | worker-2 ansible_host=10.0.20.11 6 | worker-3 ansible_host=10.0.20.12 7 | 8 | [masters] 9 | master-[1:3] 10 | 11 | [workers] 12 | worker-[1:3] 13 | 14 | [hbase_master] 15 | master-1 16 | 17 | [hbase_backup_masters] 18 | master-[2:3] 19 | 20 | [hbase_masters] 21 | master-[1:3] 22 | 23 | [hbase_workers:children] 24 | workers 25 | 26 | [hbase_rest] 27 | master-3 28 | 29 | [hbase_thrift] 30 | master-2 31 | 32 | [hbase_servers:children] 33 | hbase_master 34 | hbase_backup_masters 35 | hbase_workers 36 | hbase_rest 37 | hbase_thrift 38 | 39 | [historyserver] 40 | master-1 41 | 42 | [kdc_server] 43 | master-2 44 | 45 | [cluster:children] 46 | masters 47 | workers 48 | 49 | [zookeepers:children] 50 | masters 51 | 52 | [monitoring_collection:children] 53 | masters 54 | workers 55 | 56 | [monitoring_aggregation] 57 | master-3 58 | 59 | [monitoring_frontend] 60 | master-3 61 | -------------------------------------------------------------------------------- /cluster/ansible/roles/common/files/hosts: -------------------------------------------------------------------------------- 1 | 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 2 | ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 3 | 4 | 10.0.20.1 master-1.hbase.book master-1 5 | 10.0.20.2 master-2.hbase.book master-2 6 | 10.0.20.3 master-3.hbase.book master-3 7 | 10.0.20.10 worker-1.hbase.book worker-1 8 | 10.0.20.11 worker-2.hbase.book worker-2 9 | 10.0.20.12 worker-3.hbase.book worker-3 10 | -------------------------------------------------------------------------------- /cluster/ansible/roles/common/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- -------------------------------------------------------------------------------- /cluster/ansible/roles/hadoop/templates/etc/hadoop/conf/container-executor.cfg.j2: -------------------------------------------------------------------------------- 1 | yarn.nodemanager.linux-container-executor.group=hadoop 2 | min.user.id=500 3 | allowed.system.users=nobody,impala,hive,llama 4 | banned.users=root,hdfs,yarn,mapred,bin 5 | -------------------------------------------------------------------------------- /cluster/ansible/roles/hadoop/templates/etc/hadoop/conf/core-site.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | 18 | 19 | 20 | 21 | fs.default.name 22 | hdfs://{{ groups.masters[0] }}.{{ CLUSTER_DOMAIN }}/ 23 | 24 | 25 | hadoop.tmp.dir 26 | {{ TMP_DIR }}/hadoop-${user.name} 27 | 28 | {% if SECURITY_ENABLED %} 29 | 30 | 31 | hadoop.security.authentication 32 | kerberos 33 | 34 | 35 | hadoop.rpc.protection 36 | privacy 37 | 38 | 39 | 40 | hadoop.security.authorization 41 | false 42 | 43 | {% endif %} 44 | 45 | -------------------------------------------------------------------------------- /cluster/ansible/roles/hadoop/templates/etc/hadoop/conf/httpfs-env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. See accompanying LICENSE file. 14 | # 15 | 16 | # Set httpfs specific environment variables here. 17 | 18 | # Settings for the Embedded Tomcat that runs HttpFS 19 | # Java System properties for HttpFS should be specified in this variable 20 | # 21 | # export CATALINA_OPTS= 22 | 23 | # HttpFS logs directory 24 | # 25 | # export HTTPFS_LOG=${HTTPFS_HOME}/logs 26 | 27 | # HttpFS temporary directory 28 | # 29 | # export HTTPFS_TEMP=${HTTPFS_HOME}/temp 30 | 31 | # The HTTP port used by HttpFS 32 | # 33 | # export HTTPFS_HTTP_PORT=14000 34 | 35 | # The Admin port used by HttpFS 36 | # 37 | # export HTTPFS_ADMIN_PORT=`expr ${HTTPFS_HTTP_PORT} + 1` 38 | 39 | # The hostname HttpFS server runs on 40 | # 41 | # export HTTPFS_HTTP_HOSTNAME=`hostname -f` 42 | 43 | # Indicates if HttpFS is using SSL 44 | # 45 | # export HTTPFS_SECURITY_ENABLED=false 46 | 47 | # The location of the SSL keystore if using SSL 48 | # 49 | # export HTTPFS_SSL_KEYSTORE_FILE=${HOME}/.keystore 50 | 51 | # The password of the SSL keystore if using SSL 52 | # 53 | # export HTTPFS_SSL_KEYSTORE_PASS=password 54 | -------------------------------------------------------------------------------- /cluster/ansible/roles/hadoop/templates/etc/hadoop/conf/httpfs-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /cluster/ansible/roles/hadoop/templates/etc/hadoop/conf/mapred-env.sh.j2: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # export JAVA_HOME={{ JAVA_HOME }} 17 | 18 | export HADOOP_JOB_HISTORYSERVER_HEAPSIZE=1000 19 | 20 | export HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA 21 | 22 | #export HADOOP_JOB_HISTORYSERVER_OPTS= 23 | #export HADOOP_MAPRED_LOG_DIR="" # Where log files are stored. $HADOOP_MAPRED_HOME/logs by default. 24 | #export HADOOP_JHS_LOGGER=INFO,RFA # Hadoop JobSummary logger. 25 | #export HADOOP_MAPRED_PID_DIR= # The pid files are stored. /tmp by default. 26 | #export HADOOP_MAPRED_IDENT_STRING= #A string representing this instance of hadoop. $USER by default 27 | #export HADOOP_MAPRED_NICENESS= #The scheduling priority for daemons. Defaults to 0. 28 | -------------------------------------------------------------------------------- /cluster/ansible/roles/hadoop/templates/etc/systemd/system/datanode.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=HDFS datanode 3 | Before= 4 | After=network.target 5 | 6 | [Service] 7 | User={% if SECURITY_ENABLED %}hdfs{% else %}hadoop{% endif %} 8 | 9 | ExecStart={{ HADOOP_HOME }}/bin/hdfs datanode 10 | ExecStop=/bin/kill $MAINPID 11 | Restart=on-abort 12 | StandardOutput=journal 13 | EnvironmentFile=/etc/environment 14 | PIDFile={{ VAR_RUN_BASE }}/hadoop/pid/datanode.pid 15 | 16 | [Install] 17 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /cluster/ansible/roles/hadoop/templates/etc/systemd/system/historyserver.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=HDFS namenode 3 | Before= 4 | After=network.target 5 | 6 | [Service] 7 | User={% if SECURITY_ENABLED %}hdfs{% else %}hadoop{% endif %} 8 | 9 | ExecStart={{ HADOOP_HOME }}/bin/mapred historyserver 10 | ExecStop=/bin/kill $MAINPID 11 | Restart=on-abort 12 | StandardOutput=journal 13 | EnvironmentFile=/etc/environment 14 | PIDFile={{ VAR_RUN_BASE }}/hadoop/pid/historyserver.pid 15 | 16 | [Install] 17 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /cluster/ansible/roles/hadoop/templates/etc/systemd/system/namenode.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=HDFS namenode 3 | Before= 4 | After=network.target 5 | 6 | [Service] 7 | User={% if SECURITY_ENABLED %}hdfs{% else %}hadoop{% endif %} 8 | 9 | ExecStart={{ HADOOP_HOME }}/bin/hdfs namenode 10 | ExecStop=/bin/kill $MAINPID 11 | Restart=on-abort 12 | StandardOutput=journal 13 | EnvironmentFile=/etc/environment 14 | PIDFile={{ VAR_RUN_BASE }}/hadoop/pid/namenode.pid 15 | 16 | [Install] 17 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /cluster/ansible/roles/hadoop/templates/etc/systemd/system/nodemanager.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=YARN-Nodemanager 3 | Before= 4 | After=network.target 5 | 6 | [Service] 7 | User={% if SECURITY_ENABLED %}hdfs{% else %}hadoop{% endif %} 8 | 9 | ExecStart={{ HADOOP_HOME }}/bin/yarn nodemanager 10 | ExecStop=/bin/kill $MAINPID 11 | Restart=on-abort 12 | StandardOutput=journal 13 | EnvironmentFile=/etc/environment 14 | PIDFile={{ VAR_RUN_BASE }}/hadoop/pid/nodemanager.pid 15 | 16 | [Install] 17 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /cluster/ansible/roles/hadoop/templates/etc/systemd/system/resourcemanager.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=YARN-Resourcemanager 3 | Before= 4 | After=network.target 5 | 6 | [Service] 7 | User={% if SECURITY_ENABLED %}hdfs{% else %}hadoop{% endif %} 8 | 9 | ExecStart={{ HADOOP_HOME }}/bin/yarn resourcemanager 10 | ExecStop=/bin/kill $MAINPID 11 | Restart=on-abort 12 | StandardOutput=journal 13 | EnvironmentFile=/etc/environment 14 | PIDFile={{ VAR_RUN_BASE }}/hadoop/pid/resourcemanager.pid 15 | 16 | [Install] 17 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /cluster/ansible/roles/hbase-common/tasks/create-hbase-root.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Purpose: Create HBase root path in HDFS as privileged user 3 | # 4 | # Notes: 5 | # - Executed on single master server only. 6 | 7 | # Create HBase root directory and set access - needs a `hdfs` principal ticket 8 | 9 | - name: Use kinit to get `hdfs` ticket 10 | shell: kinit -kt /etc/security/keytabs/hdfs.keytab hdfs/master-1.hbase.book@HBASE.BOOK 11 | when: inventory_hostname == "{{ groups['hbase_master'][0] }}" 12 | 13 | - name: Determine pre-existing HBase root directory 14 | shell: "{{ HADOOP_HOME }}/bin/hdfs dfs -ls /" 15 | register: hdfs_list_root 16 | changed_when: false 17 | when: inventory_hostname == "{{ groups['hbase_master'][0] }}" 18 | 19 | - debug: var=hdfs_list_root.stdout_lines 20 | when: inventory_hostname == "{{ groups['hbase_master'][0] }}" 21 | 22 | - name: Create HBase root directory 23 | shell: "{{ HADOOP_HOME }}/bin/hdfs dfs -mkdir {{ HBASE_ROOT_DIR }}" 24 | when: inventory_hostname == "{{ groups['hbase_master'][0] }}" and hdfs_list_root.stdout.find('{{ HBASE_ROOT_DIR }}') == -1 25 | 26 | - name: Set HBase root directory access 27 | shell: "{{ HADOOP_HOME }}/bin/hdfs dfs -chown hbase:hadoop {{ HBASE_ROOT_DIR }}" 28 | when: inventory_hostname == "{{ groups['hbase_master'][0] }}" and hdfs_list_root.stdout.find('{{ HBASE_ROOT_DIR }}') == -1 29 | -------------------------------------------------------------------------------- /cluster/ansible/roles/hbase-common/templates/etc/hbase/conf/backup-masters.j2: -------------------------------------------------------------------------------- 1 | {%- for host in groups.hbase_backup_masters %} 2 | {{ host }} 3 | {% endfor -%} 4 | -------------------------------------------------------------------------------- /cluster/ansible/roles/hbase-common/templates/etc/hbase/conf/hadoop-metrics2-hbase.properties.j2: -------------------------------------------------------------------------------- 1 | # syntax: [prefix].[source|sink].[instance].[options] 2 | # See javadoc of package-info.java for org.apache.hadoop.metrics2 for details 3 | 4 | *.sink.file*.class=org.apache.hadoop.metrics2.sink.FileSink 5 | # default sampling period 6 | *.period=10 7 | 8 | hbase.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31 9 | hbase.sink.ganglia.servers=239.2.11.71:8649 10 | #hbase.sink.ganglia.servers={{ groups["workers"][0] }}:8649 11 | hbase.sink.ganglia.period=10 12 | 13 | # Below are some examples of sinks that could be used 14 | # to monitor different hbase daemons. 15 | 16 | # hbase.sink.file-all.class=org.apache.hadoop.metrics2.sink.FileSink 17 | # hbase.sink.file-all.filename=all.metrics 18 | 19 | # hbase.sink.file0.class=org.apache.hadoop.metrics2.sink.FileSink 20 | # hbase.sink.file0.context=hmaster 21 | # hbase.sink.file0.filename=master.metrics 22 | 23 | # hbase.sink.file1.class=org.apache.hadoop.metrics2.sink.FileSink 24 | # hbase.sink.file1.context=thrift-one 25 | # hbase.sink.file1.filename=thrift-one.metrics 26 | 27 | # hbase.sink.file2.class=org.apache.hadoop.metrics2.sink.FileSink 28 | # hbase.sink.file2.context=thrift-two 29 | # hbase.sink.file2.filename=thrift-one.metrics 30 | 31 | # hbase.sink.file3.class=org.apache.hadoop.metrics2.sink.FileSink 32 | # hbase.sink.file3.context=rest 33 | # hbase.sink.file3.filename=rest.metrics 34 | -------------------------------------------------------------------------------- /cluster/ansible/roles/hbase-common/templates/etc/hbase/conf/hbase-jaas.conf.j2: -------------------------------------------------------------------------------- 1 | Server { 2 | com.sun.security.auth.module.Krb5LoginModule required 3 | useKeyTab=true 4 | storeKey=true 5 | useTicketCache=false 6 | keyTab="{{ KEYTAB_DIR }}/hbase.keytab" 7 | principal="hbase/{{ inventory_hostname }}.{{ CLUSTER_DOMAIN }}@{{ KRB_REALM }}"; 8 | }; 9 | Client { 10 | com.sun.security.auth.module.Krb5LoginModule required 11 | useKeyTab=true 12 | useTicketCache=false 13 | keyTab="{{ KEYTAB_DIR }}/hbase.keytab" 14 | principal="hbase/{{ inventory_hostname }}.{{ CLUSTER_DOMAIN }}@{{ KRB_REALM }}"; 15 | }; -------------------------------------------------------------------------------- /cluster/ansible/roles/hbase-common/templates/etc/hbase/conf/hbase-policy.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 22 | 23 | 24 | 25 | security.client.protocol.acl 26 | * 27 | ACL for ClientProtocol and AdminProtocol implementations (ie. 28 | clients talking to HRegionServers) 29 | The ACL is a comma-separated list of user and group names. The user and 30 | group list is separated by a blank. For e.g. "alice,bob users,wheel". 31 | A special value of "*" means all users are allowed. 32 | 33 | 34 | 35 | security.admin.protocol.acl 36 | * 37 | ACL for HMasterInterface protocol implementation (ie. 38 | clients talking to HMaster for admin operations). 39 | The ACL is a comma-separated list of user and group names. The user and 40 | group list is separated by a blank. For e.g. "alice,bob users,wheel". 41 | A special value of "*" means all users are allowed. 42 | 43 | 44 | 45 | security.masterregion.protocol.acl 46 | * 47 | ACL for HMasterRegionInterface protocol implementations 48 | (for HRegionServers communicating with HMaster) 49 | The ACL is a comma-separated list of user and group names. The user and 50 | group list is separated by a blank. For e.g. "alice,bob users,wheel". 51 | A special value of "*" means all users are allowed. 52 | 53 | 54 | -------------------------------------------------------------------------------- /cluster/ansible/roles/hbase-common/templates/etc/hbase/conf/regionservers.j2: -------------------------------------------------------------------------------- 1 | {%- for host in groups.hbase_workers %} 2 | {{ host }} 3 | {% endfor -%} 4 | -------------------------------------------------------------------------------- /cluster/ansible/roles/hbase-master/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Purpose: Install HBase Master 3 | # 4 | # Notes: 5 | # - Executed on all dedicated servers in the cluster 6 | 7 | # Configure service script 8 | 9 | - name: Template HBase Master startup scripts 10 | template: 11 | src={{ item.src }} 12 | dest=/etc/systemd/system/{{ item.dest }} 13 | owner=root 14 | group=root 15 | mode=755 16 | backup=yes 17 | with_items: 18 | - { src: 'etc/systemd/system/hbase-master.service.j2', dest: 'hbase-master.service'} -------------------------------------------------------------------------------- /cluster/ansible/roles/hbase-master/templates/etc/systemd/system/hbase-master.service.j2: -------------------------------------------------------------------------------- 1 | #jinja2: trim_blocks:False 2 | [Unit] 3 | Description=HBase Master 4 | Before= 5 | After=network.target 6 | 7 | [Service] 8 | User=hbase 9 | EnvironmentFile=/etc/environment 10 | ExecStart={{ HBASE_HOME }}/bin/hbase master start{% if HBASE_MODE == "backup" %} -backup{% endif %} 11 | ExecStop=/bin/kill $MAINPID 12 | Restart=on-abort 13 | StandardOutput=journal 14 | PIDFile={{ PID_DIR }}/hbase-master.pid 15 | 16 | [Install] 17 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /cluster/ansible/roles/hbase-regionserver/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Purpose: Install HBase RegionServer 3 | # 4 | # Notes: 5 | # - Executed on all dedicated servers in the cluster 6 | 7 | # Configure service script 8 | 9 | - name: Template HBase RegionServer startup scripts 10 | template: 11 | src={{ item.src }} 12 | dest=/etc/systemd/system/{{ item.dest }} 13 | owner=root 14 | group=root 15 | mode=755 16 | backup=yes 17 | with_items: 18 | - { src: 'etc/systemd/system/hbase-regionserver.service.j2', dest: 'hbase-regionserver.service'} -------------------------------------------------------------------------------- /cluster/ansible/roles/hbase-regionserver/templates/etc/systemd/system/hbase-regionserver.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=HBase RegionServer 3 | Before= 4 | After=network.target 5 | 6 | [Service] 7 | User=hbase 8 | EnvironmentFile=/etc/environment 9 | ExecStart={{ HBASE_HOME }}/bin/hbase regionserver start 10 | ExecStop=/bin/kill $MAINPID 11 | Restart=on-abort 12 | StandardOutput=journal 13 | PIDFile={{ PID_DIR }}/hbase-regionserver.pid 14 | 15 | [Install] 16 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /cluster/ansible/roles/hbase-rest/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Purpose: Install HBase REST Gateway 3 | # 4 | # Notes: 5 | # - Executed on specific servers in the cluster 6 | 7 | # Configure service script 8 | 9 | - name: Install HBase REST startup scripts 10 | template: 11 | src={{ item.src }} 12 | dest=/etc/systemd/system/{{ item.dest }} 13 | owner=root 14 | group=root 15 | mode=755 16 | backup=yes 17 | with_items: 18 | - { src: "etc/systemd/system/hbase-rest.service.j2", dest: "hbase-rest.service"} -------------------------------------------------------------------------------- /cluster/ansible/roles/hbase-rest/templates/etc/systemd/system/hbase-rest.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=HBase REST Gateway 3 | Before= 4 | After=network.target 5 | 6 | [Service] 7 | User=hbase-rest 8 | EnvironmentFile=/etc/environment 9 | ExecStart={{ HBASE_HOME }}/bin/hbase rest start 10 | ExecStop=/bin/kill $MAINPID 11 | Restart=on-abort 12 | StandardOutput=journal 13 | PIDFile={{ PID_DIR }}/hbase-rest.pid 14 | 15 | [Install] 16 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /cluster/ansible/roles/hbase-thrift/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Purpose: Install HBase Thrift Gateway 3 | # 4 | # Notes: 5 | # - Executed on specific servers in the cluster 6 | 7 | # Configure service script 8 | 9 | - name: Install HBase Thrift startup scripts 10 | template: 11 | src={{ item.src }} 12 | dest=/etc/systemd/system/{{ item.dest }} 13 | owner=root 14 | group=root 15 | mode=755 16 | backup=yes 17 | with_items: 18 | - { src: "etc/systemd/system/hbase-thrift.service.j2", dest: "hbase-thrift.service"} -------------------------------------------------------------------------------- /cluster/ansible/roles/hbase-thrift/templates/etc/systemd/system/hbase-thrift.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=HBase Thrift Gateway 3 | Before= 4 | After=network.target 5 | 6 | [Service] 7 | User=hbase-thrift 8 | EnvironmentFile=/etc/environment 9 | ExecStart={{ HBASE_HOME }}/bin/hbase {% if HBASE_THRIFT_VERSION == "2" %}thrift2{% else %}thrift{% endif %} start 10 | ExecStop=/bin/kill $MAINPID 11 | Restart=on-abort 12 | StandardOutput=journal 13 | PIDFile={{ PID_DIR }}/hbase-thrift.pid 14 | 15 | [Install] 16 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /cluster/ansible/roles/kdc-server/files/etc/systemd/system/rngd.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Hardware RNG Entropy Gatherer Daemon 3 | 4 | [Service] 5 | ExecStart=/sbin/rngd -f -r /dev/urandom 6 | 7 | [Install] 8 | WantedBy=multi-user.target 9 | -------------------------------------------------------------------------------- /cluster/ansible/roles/kdc-server/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Purpose: Handler that restarts KDC daemons 3 | # 4 | 5 | - name: restart kerberos 6 | service: name={{ item }} state=restarted 7 | with_items: 8 | - krb5kdc 9 | - kadmin 10 | #when: was running already 11 | -------------------------------------------------------------------------------- /cluster/ansible/roles/kdc-server/tasks/createkdb5.yml: -------------------------------------------------------------------------------- 1 | --- 2 | #- name: Create KDB5 database 3 | # command: kdb5_util destroy -f 4 | # ignore_errors: True 5 | 6 | - name: Install rng-tools 7 | yum: name=rng-tools state=present 8 | 9 | - name: Configure rngd to use /dev/urandom since in virtualmachines there usually is no hardware source for random data 10 | copy: src=etc/systemd/system/rngd.service dest=/etc/systemd/system/rngd.service owner=root group=root 11 | 12 | - name: Start rngd service 13 | service: name=rngd state=started 14 | 15 | - name: Create key 16 | command: kdb5_util create -s -P sslsekret creates=/var/kerberos/krb5kdc/principal 17 | 18 | #- name: Stop rngd tools 19 | # service: name=rngd state=stopped 20 | 21 | #- name: Remove rng-tools 22 | # yum: name=rng-tools state=absent 23 | 24 | -------------------------------------------------------------------------------- /cluster/ansible/roles/kdc-server/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Purpose: KDC server related settings 3 | # 4 | # Notes: 5 | # - Assumes there is only one (1) KDC server node 6 | 7 | - name: Install Kerberos server package 8 | yum: name={{ item }} state=latest 9 | with_items: 10 | - krb5-server 11 | - krb5-libs 12 | 13 | - name: Copy and configure KDC server configs 14 | template: 15 | src=var/kerberos/krb5kdc/kdc.conf.j2 16 | dest=/var/kerberos/krb5kdc/kdc.conf 17 | mode=0600 18 | owner=root 19 | group=root 20 | notify: 21 | - restart kerberos 22 | 23 | - name: Copy and configure KDC server ACLs 24 | template: 25 | src=var/kerberos/krb5kdc/kadm5.acl.j2 26 | dest=/var/kerberos/krb5kdc/kadm5.acl 27 | mode=0600 28 | owner=root 29 | group=root 30 | notify: 31 | - restart kerberos 32 | 33 | - name: Copy and configure client krb5.conf 34 | template: 35 | src=etc/krb5.conf.j2 36 | dest=/etc/krb5.conf 37 | mode=0644 38 | owner=root 39 | group=root 40 | 41 | #- name: Check if KRB5 database needs to be generated 42 | # stat: path=/var/kerberos/krb5kdc/principal 43 | # register: principal_file 44 | 45 | - name: Create KRB5 database if necessary 46 | include: createkdb5.yml 47 | # when: (principal_file.stat.exists is defined and not principal_file.stat.exists) or principal_file.stat.exists is not defined 48 | 49 | - name: Start Kerberos services 50 | service: name={{ item }} state=started enabled=yes 51 | with_items: 52 | - krb5kdc 53 | - kadmin 54 | 55 | - name: Create principals 56 | include: createprincs.yml 57 | 58 | -------------------------------------------------------------------------------- /cluster/ansible/roles/kdc-server/templates/etc/krb5.conf.j2: -------------------------------------------------------------------------------- 1 | [logging] 2 | default = FILE:/var/log/krb5libs.log 3 | kdc = FILE:/var/log/krb5kdc.log 4 | admin_server = FILE:/var/log/kadmind.log 5 | 6 | [libdefaults] 7 | default_realm = {{ KRB_REALM }} 8 | dns_lookup_realm = false 9 | dns_lookup_kdc = false 10 | ticket_lifetime = 24h 11 | renew_lifetime = 7d 12 | forwardable = true 13 | 14 | [realms] 15 | {{ KRB_REALM }} = { 16 | kdc = {{ KRB_SERVER }} 17 | admin_server = {{ KRB_SERVER }} 18 | } 19 | 20 | [domain_realm] 21 | .{{ CLUSTER_DOMAIN }} = {{ KRB_REALM }} 22 | {{ KRB_SERVER }} = {{ KRB_REALM }} 23 | 24 | -------------------------------------------------------------------------------- /cluster/ansible/roles/kdc-server/templates/var/kerberos/krb5kdc/.k5.INTERNAL.LARSGEORGE.COM: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/larsgeorge/hbase-book/2ecb9a13820e7effb12c4ae1cb998eddc1414e46/cluster/ansible/roles/kdc-server/templates/var/kerberos/krb5kdc/.k5.INTERNAL.LARSGEORGE.COM -------------------------------------------------------------------------------- /cluster/ansible/roles/kdc-server/templates/var/kerberos/krb5kdc/kadm5.acl.j2: -------------------------------------------------------------------------------- 1 | */admin@{{ KRB_REALM }} * 2 | -------------------------------------------------------------------------------- /cluster/ansible/roles/kdc-server/templates/var/kerberos/krb5kdc/kdc.conf.j2: -------------------------------------------------------------------------------- 1 | [kdcdefaults] 2 | kdc_ports = 88 3 | kdc_tcp_ports = 88 4 | 5 | [realms] 6 | {{ KRB_REALM }} = { 7 | #master_key_type = aes256-cts 8 | acl_file = /var/kerberos/krb5kdc/kadm5.acl 9 | dict_file = /usr/share/dict/words 10 | admin_keytab = /var/kerberos/krb5kdc/kadm5.keytab 11 | default_principal_flags = renewable 12 | max_renewable_life = 7d 13 | 14 | supported_enctypes = aes256-cts:normal aes128-cts:normal des3-hmac-sha1:normal arcfour-hmac:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal 15 | } 16 | -------------------------------------------------------------------------------- /cluster/ansible/roles/krb-client/files/UnlimitedJCEPolicy/US_export_policy.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/larsgeorge/hbase-book/2ecb9a13820e7effb12c4ae1cb998eddc1414e46/cluster/ansible/roles/krb-client/files/UnlimitedJCEPolicy/US_export_policy.jar -------------------------------------------------------------------------------- /cluster/ansible/roles/krb-client/files/UnlimitedJCEPolicy/local_policy.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/larsgeorge/hbase-book/2ecb9a13820e7effb12c4ae1cb998eddc1414e46/cluster/ansible/roles/krb-client/files/UnlimitedJCEPolicy/local_policy.jar -------------------------------------------------------------------------------- /cluster/ansible/roles/krb-client/files/certs/rootCA.key: -------------------------------------------------------------------------------- 1 | -----BEGIN ENCRYPTED PRIVATE KEY----- 2 | MIIFDjBABgkqhkiG9w0BBQ0wMzAbBgkqhkiG9w0BBQwwDgQIUU2QPzsYmeMCAggA 3 | MBQGCCqGSIb3DQMHBAjSnfHF/4a7sASCBMgmmCkPZTz6Lo1RrQjJxHgu9wMYbd0c 4 | cuxH7uR6OCxwtSnS1h1znweuT+zXOSAQilSXnvlZVWN9MF0OYNUODi5kZGm14WzC 5 | F2UQE2OMJ56vCqhDUar4oqnjyU0OzhXOiXPVH+G2kfQfrCDknkzthBqdl+HIkAyk 6 | 99Ck4vGQ6+b2luW8G5lDBSIwg8NUpyJXYjB4KBhl3woZg6rhdYLyVDm1zekm0jIK 7 | ibUIS67FghmhDxFVbRStZx5VC6yRCSjl+icqXTVR/Y36sg55m9iUfwjBrT9wDTLu 8 | JCRz9ObTuoSzOK4uNz2w8vZekKUR4RXf3ZlBaP9G1hgIXl1IAX7kYiOvDgurr41O 9 | O/7qskGBLD2OGBrKGbT+utU2otiQUJspcWgNeBbD5EhzfIa9n65kJ+xfUv3i+akC 10 | f/MRN4w5h8oMch8qms0WHAnBCCfq6SRBZwBWMDeH6UFFZtvaXeEASxDr3UI3wjcv 11 | 0nrcSExwyrC1UE22KF3cSbIyKvNxFLh3KpiTLs7HxUjKCLHHt6Q4fkFZrv3K+PR6 12 | 9IGJSC/VN6GdJxavxSMhX8B0gIYFBaqmpRx6exjRTAuiLeYcdEKRAu8ESQQLnQ/0 13 | qPsfuBmBUb+lg6z5MyUsL6uwSL6MPg1Tct4k4r0LH4NNUC1s1lSwI0+UeQo0BsiD 14 | A+uGwilpUrt6FoI6Fn/zPW+PL/aFy0c50X74Rl96F37U5OuSGncbrb8zTL79ijNK 15 | b71se5gwJnL9ZcRRjzaIaEvvaq0OC2N3zQ7SHph7h45Kcrg6riwKpSXaiaXLg5od 16 | sV/GrikT5v/2q6m6kvJztFrncQdCTC0cjCa1B8PugvM3OIKvdTxQlrmduikvgKeG 17 | gV4tyASxK8hG3nPzhSKIFtyPROYScrUNxEH/0FNbZ72ULyT11Qb4IJLIrWOfulF2 18 | 3R4x4BvbA3FndzJp9uaOhkftCc2dc1+nyLZhi9NglVSEKnOT8ac94IWTWpqlFIfk 19 | OARZ49t/qd53a+XG+U21mW5lKOoQCfXNCXXRiREmaTtHUzFHZ21lzZNghkaU4e6P 20 | FA3Oo09DOtyi3sKkLg+rpz7gMXXtasP1LVxm6+xmI4lp+mD8OnriyTE3yaQutl89 21 | zBs3ynrKjmqmxBGcJpO0/ap6J+dPaVVTnxRdgatsUQacA7X9fEwPn2HyuMr9p2Zm 22 | V/X6TSqhXM5pyYiIu5S247lQQcFuKEkFHoqFZkRXrtRZUz/5RRa/FDQsbtnCu2Tm 23 | GcJH6BYZXL1aVFbC320HrOI69VlzZONjR0h/o6AEVQPR8Z0zJuWD6SQgfRej/6g+ 24 | paJWhEUPXaR5jScK2/SRVRNhCH5cXE69UWjyhqp3q8gbQpuIjDT6dFhKsNlzclam 25 | RcHpOxEggejHuPAMUHdzh48jg43mMqIwsbvKdNLsWwuVhATQWi25jqEG8w0lYpxk 26 | cOge0GARzMV1KThnmetl2weQSsd/JIJpRcsuLiShcxzplhr8AXAh1gqZ3Vr+QjUI 27 | vl1JMoIfKABbRPSX/Mw2FaFbulrClDGNMjeENc76WbpOi7dJoRrtnzR8oLxj/WfM 28 | ZsJnCGHk44GLIah4d47CQTM1gl7zpCG79Yn6/WUD2vhGHH99dHfJ0ZxfMYTjfrZd 29 | kWc= 30 | -----END ENCRYPTED PRIVATE KEY----- 31 | -------------------------------------------------------------------------------- /cluster/ansible/roles/krb-client/files/certs/rootCA.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDozCCAougAwIBAgIJAMcycd2mmUR3MA0GCSqGSIb3DQEBBQUAMGgxCzAJBgNV 3 | BAYTAkRFMQ4wDAYDVQQHDAVXZWRlbDEfMB0GA1UECgwWT3BlbkNvcmUgR21iSCAm 4 | IENvLiBLRzERMA8GA1UECwwISW50ZXJuYWwxFTATBgNVBAMMDG9wZW5jb3JlLmNv 5 | bTAeFw0xNjAyMjQwNzAyNDNaFw0xODExMjAwNzAyNDNaMGgxCzAJBgNVBAYTAkRF 6 | MQ4wDAYDVQQHDAVXZWRlbDEfMB0GA1UECgwWT3BlbkNvcmUgR21iSCAmIENvLiBL 7 | RzERMA8GA1UECwwISW50ZXJuYWwxFTATBgNVBAMMDG9wZW5jb3JlLmNvbTCCASIw 8 | DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALIpr1FbKq+pIE4B+6Qw3LClFJNX 9 | UcbMjq7Rjx3Fx0RSfohGrURbkkjk4uebdNj8K23RjLI82z4QRmatESiS2H0KX1Hy 10 | BpwMwzwLgf9GVOje2ij02S2Dm9itMgOmUwmgfTDA7FDVUsjeXIZ/Ug3P824paqvS 11 | uAe5Ji5JKcLixoVvLfc+3WAxEBu2JaVByFbCLQSAuNpyBKAfEzlUGvvUiWADzAY7 12 | cgvdPDnmCLroi1QqnV2pfSqsNeeBx+sDHPQ/e4RcVkuLBxXwrmnmsMyyyVA0dGYE 13 | QXBmR/XfHY58vIYXZ1SCfrhmqqfG4K4o1v/d2hEeiWvM/9bM5cHQhpGZuscCAwEA 14 | AaNQME4wHQYDVR0OBBYEFMjF2Myi0vTaK3dGJ2FLum7IsspCMB8GA1UdIwQYMBaA 15 | FMjF2Myi0vTaK3dGJ2FLum7IsspCMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEF 16 | BQADggEBAE5MFibNrITJWlcRLKeVGLJhWqgDaT6/qhrITWmjhxWlicxPTV1nJHta 17 | 8jTLoA243uW/LkOln8CNgZZ+XxoHfyPI8u88EZHysq9hetkT0Uv7eBEY6uaMAHSO 18 | bpClieKKyxZa+jz1Wsee84b8s07FbBvQaH4OdWt71EqDqlLxt5mAV0j4bxWp/+lK 19 | RsrG7tqi8j5AU3Tk1ZceJkUQ48bMKCM3UndDbaPl14EQgSk7ax5M9XWOi5BLULgl 20 | 9b2qw+mJ3WRUV1/J6HTT3g7LRaL6REDsVOGI+1T1zZ3Y4CTl+OOBE7OtbT6m2Mwg 21 | KoNtkwRbFjWcB8fU6+xl96F1zLmVMsg= 22 | -----END CERTIFICATE----- 23 | -------------------------------------------------------------------------------- /cluster/ansible/roles/krb-client/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- -------------------------------------------------------------------------------- /cluster/ansible/roles/krb-client/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Purpose: Add Kerberos rerlated traits to the nodes 3 | # 4 | # Notes: 5 | # - Executed on all servers in the cluster 6 | - name: Create Hadoop related directories (secure mode) 7 | file: 8 | path={{ KEYTAB_DIR }} 9 | state=directory 10 | owner=root 11 | group=root 12 | mode=755 13 | recurse=yes 14 | 15 | - name: Install JCE libraries 16 | copy: 17 | src=UnlimitedJCEPolicy/ 18 | dest=/etc/alternatives/jre/lib/security/ 19 | owner=root 20 | group=root 21 | mode=644 22 | backup=yes 23 | 24 | - name: Install Kerberos client packages 25 | yum: name={{ item }} state=present # update_cache=yes 26 | with_items: 27 | - krb5-workstation 28 | - libselinux-python 29 | 30 | - name: Copy and configure client krb5.conf 31 | template: 32 | src=etc/krb5.conf.j2 33 | dest=/etc/krb5.conf 34 | mode=0644 35 | owner=root 36 | group=root 37 | 38 | - name: Upload keytabs 39 | copy: 40 | src=files/keytabs/{{ item }}-{{ inventory_hostname }}.keytab 41 | dest={{ KEYTAB_DIR }}/{{ item }}.keytab 42 | with_items: 43 | - "{{ KRB_SERVICES_WITH_HTTP }}" 44 | - copy: 45 | src=files/keytabs/{{ item.user }}.keytab 46 | dest={{ KEYTAB_DIR }}/{{ item.user }}.keytab 47 | with_items: 48 | - "{{ END_USERS }}" 49 | -------------------------------------------------------------------------------- /cluster/ansible/roles/krb-client/templates/etc/krb5.conf.j2: -------------------------------------------------------------------------------- 1 | [logging] 2 | default = FILE:/var/log/krb5libs.log 3 | kdc = FILE:/var/log/krb5kdc.log 4 | admin_server = FILE:/var/log/kadmind.log 5 | 6 | [libdefaults] 7 | default_realm = {{ KRB_REALM }} 8 | dns_lookup_realm = false 9 | dns_lookup_kdc = false 10 | ticket_lifetime = 24h 11 | renew_lifetime = 7d 12 | forwardable = true 13 | 14 | [realms] 15 | {{ KRB_REALM }} = { 16 | kdc = {{ KRB_SERVER }} 17 | admin_server = {{ KRB_SERVER }} 18 | } 19 | 20 | [domain_realm] 21 | .{{ CLUSTER_DOMAIN }} = {{ KRB_REALM }} 22 | {{ KRB_SERVER }} = {{ KRB_REALM }} 23 | 24 | -------------------------------------------------------------------------------- /cluster/ansible/roles/monitoring-aggregation/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Restart monitoring aggregation daemons 3 | service: name=gmetad state=restarted -------------------------------------------------------------------------------- /cluster/ansible/roles/monitoring-aggregation/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Purpose: Monitoring aggregation related settings 3 | # 4 | # Notes: 5 | # - Executed on specific servers in the cluster 6 | # - Installs agents/daemons that aggregate the system metrics 7 | 8 | - name: Install monitoring aggregation packages 9 | yum: name={{ item }} state=present # update_cache=yes 10 | with_items: 11 | - ganglia-gmetad 12 | when: MONITORING_ENABLED == true 13 | 14 | - name: Configure monitoring aggregation environment 15 | template: src={{ item.src }} dest={{ item.dest }} owner=root group=root mode=0644 16 | with_items: 17 | - { src: "etc/ganglia/gmetad.conf.j2", dest: "/etc/ganglia/gmetad.conf" } 18 | when: MONITORING_ENABLED == true 19 | notify: 20 | - Restart monitoring aggregation daemons 21 | 22 | - name: Start monitoring aggregation services 23 | service: name={{ item }} state=started enabled=yes 24 | with_items: 25 | - gmetad 26 | when: MONITORING_ENABLED == true 27 | -------------------------------------------------------------------------------- /cluster/ansible/roles/monitoring-collection/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Restart monitoring collection daemons 3 | service: name=gmond state=restarted -------------------------------------------------------------------------------- /cluster/ansible/roles/monitoring-collection/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Purpose: Monitoring collection related settings 3 | # 4 | # Notes: 5 | # - Executed (usually) on all servers in the cluster 6 | # - Installs agents/daemons that collect the system metrics 7 | 8 | - name: Install monitoring collection packages 9 | yum: name={{ item }} state=present # update_cache=yes 10 | with_items: 11 | - ganglia-gmond 12 | when: MONITORING_ENABLED == true 13 | 14 | # cluster { 15 | # name = "unspecified" 16 | # owner = "unspecified" 17 | # latlong = "unspecified" 18 | # url = "unspecified" 19 | # } 20 | # 21 | # host { 22 | # location = "unspecified" 23 | # } 24 | 25 | - name: Configure monitoring environment 26 | template: src={{ item.src }} dest={{ item.dest }} owner=root group=root mode=0644 27 | with_items: 28 | - { src: "etc/ganglia/gmond.conf.j2", dest: "/etc/ganglia/gmond.conf" } 29 | when: MONITORING_ENABLED == true 30 | notify: 31 | - Restart monitoring collection daemons 32 | 33 | #- name: Modify monitoring collection daemon configuration 34 | # lineinfile: dest="/etc/ganglia/gmond.conf" regexp=' name = "unspecified"' line=' name = "HBase Cluster"' 35 | # lineinfile: dest="/etc/ganglia/gmond.conf" regexp=' owner = "unspecified"' line=' owner = "HBase Book Project"' 36 | 37 | - name: Start monitoring collection services 38 | service: name={{ item }} state=started enabled=yes 39 | when: MONITORING_ENABLED == true 40 | with_items: 41 | - gmond 42 | -------------------------------------------------------------------------------- /cluster/ansible/roles/monitoring-frontend/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Restart monitoring frontend daemons 3 | service: name=httpd state=restarted -------------------------------------------------------------------------------- /cluster/ansible/roles/monitoring-frontend/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Purpose: Monitoring frontend related settings 3 | # 4 | # Notes: 5 | # - Executed on a specific server in the cluster 6 | # - Installs web frontend that displays the system metrics 7 | 8 | - name: Install monitoring frontend packages 9 | yum: name={{ item }} state=present # update_cache=yes 10 | with_items: 11 | - ganglia-web # installs httpd etc. as dependencies 12 | when: MONITORING_ENABLED == true 13 | 14 | - name: Allow access to frontend from host network 15 | template: src={{ item.src }} dest={{ item.dest }} owner=root group=root mode=0644 16 | with_items: 17 | - { src: "etc/httpd/conf.d/ganglia.conf.j2", dest: "/etc/httpd/conf.d/ganglia.conf" } 18 | when: MONITORING_ENABLED == true 19 | notify: 20 | - Restart monitoring frontend daemons 21 | 22 | - name: Start monitoring frontend services 23 | service: name={{ item }} state=started enabled=yes 24 | with_items: 25 | - httpd 26 | when: MONITORING_ENABLED == true 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /cluster/ansible/roles/monitoring-frontend/templates/etc/httpd/conf.d/ganglia.conf.j2: -------------------------------------------------------------------------------- 1 | # 2 | # Ganglia monitoring system php web frontend 3 | # 4 | 5 | Alias /ganglia /usr/share/ganglia 6 | 7 | 8 | Order deny,allow 9 | Deny from all 10 | Allow from 127.0.0.1 11 | Allow from ::1 12 | # Allow from HBase Cluster host network 13 | Allow from 10. 14 | # Allow from .example.com 15 | 16 | -------------------------------------------------------------------------------- /cluster/ansible/roles/ssl-phase-1/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Purpose: Creates the TLS/SSL certificates needed 3 | # 4 | # Notes: 5 | # - Executed on every master and worker server 6 | # - This is part I of a three step process: 7 | # Phase 1) Generate all certs, keystores, and CSRs for every server and process 8 | # Phase 2) Sign all CSRs with the central CA root certificate 9 | # Phase 3) Import all signed CSRs and put shared truststore in place 10 | 11 | - name: Create keystore staging directory in root account (if not already present) 12 | file: 13 | path={{ KEYSTORE_STAGING_DIR }} 14 | state=directory 15 | owner=root 16 | group=root 17 | mode=750 18 | 19 | - name: Create keystore (if not already present) 20 | command: /usr/bin/keytool -genkeypair -keystore {{ KEYSTORE_STAGING_DIR }}/{{ inventory_hostname }}.jks -keyalg RSA -alias {{ inventory_hostname }} -dname "CN={{ inventory_hostname }}" -storepass "{{ SSL_STORE_PASSWORD }}" -keypass "{{ SSL_KEY_PASSWORD }}" -validity 1000 21 | args: 22 | creates: "{{ KEYSTORE_STAGING_DIR }}/{{ inventory_hostname }}.jks" 23 | 24 | - name: Export CSR (if not already present) 25 | command: /usr/bin/keytool -certreq -keystore {{ KEYSTORE_STAGING_DIR }}/{{ inventory_hostname }}.jks -alias {{ inventory_hostname }} -storepass "{{ SSL_STORE_PASSWORD }}" -keypass "{{ SSL_KEY_PASSWORD }}" -file {{ KEYSTORE_STAGING_DIR }}/{{ inventory_hostname }}.csr 26 | args: 27 | creates: "{{ KEYSTORE_STAGING_DIR }}/{{ inventory_hostname }}.csr" 28 | 29 | - name: Set access rights for keystore and CSR 30 | file: 31 | path={{ KEYSTORE_STAGING_DIR }}/{{ inventory_hostname }}.{{ item }} 32 | state=file 33 | owner=hadoop 34 | group=hadoop 35 | mode=440 36 | with_items: 37 | - [ 'jks', 'csr' ] 38 | 39 | - name: Fetch CSR for later signing (in phase 2) 40 | fetch: src={{ KEYSTORE_STAGING_DIR }}/{{ inventory_hostname }}.csr dest=files/tmp/certs/ flat=yes 41 | -------------------------------------------------------------------------------- /cluster/ansible/roles/zookeeper/templates/data/zookeeper/myid.j2: -------------------------------------------------------------------------------- 1 | {{ ZOOKEEPER_ID }} -------------------------------------------------------------------------------- /cluster/ansible/roles/zookeeper/templates/etc/systemd/system/zookeeper.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Zookeeper 3 | Before= 4 | After=network.target 5 | 6 | [Service] 7 | User=zookeeper 8 | ExecStart={{ ZOOKEEPER_HOME }}/bin/zkServer.sh start-foreground {{ ZOOKEEPER_CONFIG }}/zoo.cfg 9 | ExecStop=/bin/kill $MAINPID 10 | Restart=on-abort 11 | StandardOutput=journal 12 | EnvironmentFile={{ ZOOKEEPER_CONFIG }}/java.env 13 | PIDFile={{ VAR_RUN_BASE }}/zookeeper/pid/zookeeper.pid 14 | 15 | [Install] 16 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /cluster/ansible/roles/zookeeper/templates/etc/zookeeper/jaas.conf.j2: -------------------------------------------------------------------------------- 1 | Server { 2 | com.sun.security.auth.module.Krb5LoginModule required 3 | useKeyTab=true 4 | keyTab="{{ KEYTAB_DIR }}/zookeeper.keytab" 5 | storeKey=true 6 | useTicketCache=false 7 | principal="zookeeper/{{ inventory_hostname }}.{{ CLUSTER_DOMAIN }}@{{ KRB_REALM }}"; 8 | }; -------------------------------------------------------------------------------- /cluster/ansible/roles/zookeeper/templates/etc/zookeeper/java.env.j2: -------------------------------------------------------------------------------- 1 | JVMFLAGS="-Djava.security.auth.login.config={{ ZOOKEEPER_CONFIG }}/jaas.conf" 2 | -------------------------------------------------------------------------------- /cluster/ansible/roles/zookeeper/templates/etc/zookeeper/log4j.properties.j2: -------------------------------------------------------------------------------- 1 | # Define some default values that can be overridden by system properties 2 | zookeeper.root.logger=INFO, CONSOLE 3 | zookeeper.console.threshold=INFO 4 | zookeeper.log.dir={{ VAR_RUN_BASE }}/zookeeper/logs 5 | zookeeper.log.file=zookeeper.log 6 | zookeeper.log.threshold=DEBUG 7 | zookeeper.tracelog.dir=. 8 | zookeeper.tracelog.file=zookeeper_trace.log 9 | 10 | # 11 | # ZooKeeper Logging Configuration 12 | # 13 | 14 | # Format is " (, )+ 15 | 16 | # DEFAULT: console appender only 17 | log4j.rootLogger=${zookeeper.root.logger} 18 | 19 | # Example with rolling log file 20 | #log4j.rootLogger=DEBUG, CONSOLE, ROLLINGFILE 21 | 22 | # Example with rolling log file and tracing 23 | #log4j.rootLogger=TRACE, CONSOLE, ROLLINGFILE, TRACEFILE 24 | 25 | # 26 | # Log INFO level and above messages to the console 27 | # 28 | log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender 29 | log4j.appender.CONSOLE.Threshold=${zookeeper.console.threshold} 30 | log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout 31 | log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n 32 | 33 | # 34 | # Add ROLLINGFILE to rootLogger to get log file output 35 | # Log DEBUG level and above messages to a log file 36 | log4j.appender.ROLLINGFILE=org.apache.log4j.RollingFileAppender 37 | log4j.appender.ROLLINGFILE.Threshold=${zookeeper.log.threshold} 38 | log4j.appender.ROLLINGFILE.File=${zookeeper.log.dir}/${zookeeper.log.file} 39 | 40 | # Max log file size of 10MB 41 | log4j.appender.ROLLINGFILE.MaxFileSize=10MB 42 | # uncomment the next line to limit number of backup files 43 | #log4j.appender.ROLLINGFILE.MaxBackupIndex=10 44 | 45 | log4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout 46 | log4j.appender.ROLLINGFILE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n 47 | 48 | 49 | # 50 | # Add TRACEFILE to rootLogger to get log file output 51 | # Log DEBUG level and above messages to a log file 52 | log4j.appender.TRACEFILE=org.apache.log4j.FileAppender 53 | log4j.appender.TRACEFILE.Threshold=TRACE 54 | log4j.appender.TRACEFILE.File=${zookeeper.tracelog.dir}/${zookeeper.tracelog.file} 55 | 56 | log4j.appender.TRACEFILE.layout=org.apache.log4j.PatternLayout 57 | ### Notice we are including log4j's NDC here (%x) 58 | log4j.appender.TRACEFILE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L][%x] - %m%n 59 | -------------------------------------------------------------------------------- /cluster/ansible/roles/zookeeper/templates/etc/zookeeper/zoo.cfg.j2: -------------------------------------------------------------------------------- 1 | # The number of milliseconds of each tick 2 | tickTime=2000 3 | # The number of ticks that the initial 4 | # synchronization phase can take 5 | initLimit=10 6 | # The number of ticks that can pass between 7 | # sending a request and getting an acknowledgement 8 | syncLimit=5 9 | # The directory where the snapshot is stored. 10 | dataDir={{ ZOOKEEPER_DATA }} 11 | # The port at which the clients will connect 12 | clientPort=2181 13 | # The servers in the ensemble 14 | {% for host in groups.zookeepers %} 15 | server.{{ loop.index }}={{ host }}.{{ CLUSTER_DOMAIN }}:2888:3888 16 | {% endfor %} 17 | {%- if SECURITY_ENABLED %} 18 | authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider 19 | jaasLoginRenew=3600000 20 | kerberos.removeHostFromPrincipal=true 21 | kerberos.removeRealmFromPrincipal=true 22 | {% endif -%} 23 | -------------------------------------------------------------------------------- /cluster/ansible/sslcerts.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Special playbook to install only SSL related details. 3 | 4 | - name: SSL Phase 1 (Prepare Servers) 5 | hosts: all 6 | remote_user: larsgeorge 7 | become: yes 8 | 9 | roles: 10 | - { role: ssl-phase-1, when: SECURITY_ENABLED == true } 11 | 12 | - name: SSL Phase 2 (Sign CSRs) 13 | hosts: kdc_server 14 | remote_user: larsgeorge 15 | become: yes 16 | 17 | roles: 18 | - { role: ssl-phase-2, when: SECURITY_ENABLED == true } 19 | 20 | - name: SSL Phase 3 (Install CSRs and Truststores) 21 | hosts: all 22 | remote_user: larsgeorge 23 | become: yes 24 | 25 | roles: 26 | - { role: ssl-phase-3, when: SECURITY_ENABLED == true } 27 | 28 | -------------------------------------------------------------------------------- /common/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | 6 | com.larsgeorge 7 | hbase-book 8 | 2.0 9 | ../pom.xml 10 | 11 | 12 | com.larsgeorge 13 | hbase-book-common 14 | 2.0 15 | jar 16 | 17 | HBase Book Common Code 18 | 19 | -------------------------------------------------------------------------------- /hush/.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.iml 3 | target/ 4 | .idea/ 5 | .classpath 6 | .project 7 | -------------------------------------------------------------------------------- /hush/.settings/org.eclipse.jdt.ui.prefs: -------------------------------------------------------------------------------- 1 | #Tue May 03 23:50:35 EDT 2011 2 | eclipse.preferences.version=1 3 | formatter_profile=_HBase Book 4 | formatter_settings_version=11 5 | -------------------------------------------------------------------------------- /hush/src/main/java/com/hbasebook/hush/coprocessor/FetchUrlInfoObserver.java: -------------------------------------------------------------------------------- 1 | package com.hbasebook.hush.coprocessor; 2 | 3 | import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver; 4 | import org.apache.hadoop.hbase.coprocessor.RegionObserver; 5 | 6 | import java.io.IOException; 7 | 8 | /** 9 | * An observer implementation that checks for new URL records and 10 | * asynchronously fetches some into about it. 11 | */ 12 | public class FetchUrlInfoObserver extends BaseRegionObserver 13 | implements RegionObserver, FetchUrlInfoProtocol { 14 | 15 | 16 | @Override 17 | public void processQueue() { 18 | } 19 | 20 | @Override 21 | public void updateUrl(String url) throws IOException { 22 | } 23 | 24 | @Override 25 | public void updateUrlMD5(String urlMD5) throws IOException { 26 | } 27 | 28 | // @Override 29 | // public long getProtocolVersion(String s, long l) throws IOException { 30 | // return 1L; 31 | // } 32 | } 33 | -------------------------------------------------------------------------------- /hush/src/main/java/com/hbasebook/hush/coprocessor/FetchUrlInfoProtocol.java: -------------------------------------------------------------------------------- 1 | package com.hbasebook.hush.coprocessor; 2 | 3 | //import org.apache.hadoop.hbase.ipc.CoprocessorProtocol; 4 | 5 | import java.io.IOException; 6 | 7 | /** 8 | * Defines a dynamic RPC call to initiate server side actions. 9 | */ 10 | public interface FetchUrlInfoProtocol { //extends CoprocessorProtocol { 11 | 12 | void processQueue(); 13 | 14 | void updateUrl(String url) throws IOException; 15 | void updateUrlMD5(String urlMD5) throws IOException; 16 | } 17 | -------------------------------------------------------------------------------- /hush/src/main/java/com/hbasebook/hush/model/ColumnQualifier.java: -------------------------------------------------------------------------------- 1 | package com.hbasebook.hush.model; 2 | 3 | import java.text.ParseException; 4 | import java.text.SimpleDateFormat; 5 | import java.util.Date; 6 | 7 | import org.apache.hadoop.hbase.util.Bytes; 8 | 9 | import com.hbasebook.hush.ResourceManager; 10 | 11 | /** 12 | * The column qualifiers for the statistics table. 13 | */ 14 | public enum ColumnQualifier { 15 | DAY("yyyyMMdd", TimeFrame.DAY), 16 | WEEK("yyyyww", TimeFrame.WEEK), 17 | MONTH("yyyyMM", TimeFrame.MONTH); 18 | 19 | private final SimpleDateFormat formatter; 20 | private final TimeFrame timeFrame; 21 | 22 | ColumnQualifier(String format, TimeFrame timeFrame) { 23 | this.formatter = new SimpleDateFormat(format); 24 | this.timeFrame = timeFrame; 25 | } 26 | 27 | public byte[] getColumnName(Date date, StatisticsCategory type) { 28 | return Bytes.add(Bytes.toBytes(formatter.format(date)), 29 | ResourceManager.ZERO, new byte[]{type.getCode()}); 30 | } 31 | 32 | public TimeFrame getTimeFrame() { 33 | return timeFrame; 34 | } 35 | 36 | public Date parseDate(String date) throws ParseException { 37 | return formatter.parse(date); 38 | } 39 | } -------------------------------------------------------------------------------- /hush/src/main/java/com/hbasebook/hush/model/Counter.java: -------------------------------------------------------------------------------- 1 | package com.hbasebook.hush.model; 2 | 3 | /** 4 | * Helper to sort counters by key or value. 5 | * 6 | * @param The key type. 7 | * @param The value type. 8 | */ 9 | public class Counter, V extends Comparable> 10 | implements Comparable> { 11 | 12 | public enum Sort { 13 | KeyAsc, KeyDesc, ValueAsc, ValueDesc; 14 | } 15 | 16 | private K key; 17 | private V value; 18 | private Sort sort; 19 | 20 | public Counter(K key, V value) { 21 | this(key, value, Sort.KeyAsc); 22 | } 23 | 24 | public Counter(K key, V value, Sort sort) { 25 | this.key = key; 26 | this.value = value; 27 | this.sort = sort; 28 | } 29 | 30 | public K getKey() { 31 | return key; 32 | } 33 | 34 | public void setKey(K key) { 35 | this.key = key; 36 | } 37 | 38 | public V getValue() { 39 | return value; 40 | } 41 | 42 | public void setValue(V value) { 43 | this.value = value; 44 | } 45 | 46 | public Sort getSort() { 47 | return sort; 48 | } 49 | 50 | public void setSort(Sort sort) { 51 | this.sort = sort; 52 | } 53 | 54 | @Override 55 | public int compareTo(Counter other) { 56 | switch (sort) { 57 | case KeyDesc: 58 | return other.key.compareTo(key); 59 | case ValueAsc: 60 | return value.compareTo(other.value); 61 | case ValueDesc: 62 | return other.value.compareTo(value); 63 | default: 64 | return key.compareTo(other.key); 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /hush/src/main/java/com/hbasebook/hush/model/LongUrl.java: -------------------------------------------------------------------------------- 1 | package com.hbasebook.hush.model; 2 | 3 | public class LongUrl { 4 | private final String url; 5 | private final String shortId; 6 | 7 | public LongUrl(String url, String shortId) { 8 | this.url = url; 9 | this.shortId = shortId; 10 | } 11 | 12 | public String getUrl() { 13 | return url; 14 | } 15 | 16 | public String getShortId() { 17 | return shortId; 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /hush/src/main/java/com/hbasebook/hush/model/ShortDomain.java: -------------------------------------------------------------------------------- 1 | package com.hbasebook.hush.model; 2 | 3 | import java.util.List; 4 | 5 | public class ShortDomain { 6 | private final String shortDomain; 7 | private final List domains; 8 | 9 | public ShortDomain(String shortDomain, List domains) { 10 | super(); 11 | this.shortDomain = shortDomain; 12 | this.domains = domains; 13 | } 14 | 15 | public String getShortDomain() { 16 | return shortDomain; 17 | } 18 | 19 | public List getDomains() { 20 | return domains; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /hush/src/main/java/com/hbasebook/hush/model/ShortUrl.java: -------------------------------------------------------------------------------- 1 | package com.hbasebook.hush.model; 2 | 3 | import com.hbasebook.hush.ResourceManager; 4 | 5 | /** 6 | * Container for all the information needed for a shortened URL. 7 | */ 8 | public class ShortUrl { 9 | private final String id; 10 | private final String domain; 11 | private final String longUrl; 12 | private final String refShortId; 13 | private final String user; 14 | private final long clicks; 15 | 16 | public ShortUrl(String id, String domain, String longUrl, 17 | String refShortId, String user) { 18 | this(id, domain, longUrl, refShortId, user, 0); 19 | } 20 | 21 | public ShortUrl(String id, String domain, String longUrl, 22 | String refShortId, String user, long clicks) { 23 | this.id = id; 24 | this.domain = domain; 25 | this.longUrl = longUrl; 26 | this.refShortId = refShortId; 27 | this.user = user; 28 | this.clicks = clicks; 29 | } 30 | 31 | public String getId() { 32 | return id; 33 | } 34 | 35 | public String getDomain() { 36 | return domain; 37 | } 38 | 39 | public String getLongUrl() { 40 | return longUrl; 41 | } 42 | 43 | public String getRefShortId() { 44 | return refShortId; 45 | } 46 | 47 | public String getUser() { 48 | return user; 49 | } 50 | 51 | public long getClicks() { 52 | return clicks; 53 | } 54 | 55 | @Override 56 | public String toString() { 57 | int port = ResourceManager.getHushPort(); 58 | return "http://" + domain + (port != 80 ? ":" + port : "") + "/" + id; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /hush/src/main/java/com/hbasebook/hush/model/ShortUrlStatistics.java: -------------------------------------------------------------------------------- 1 | package com.hbasebook.hush.model; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | import java.util.NavigableSet; 6 | 7 | /** 8 | * Container class to hold the details for the presentation layer. 9 | */ 10 | public class ShortUrlStatistics { 11 | private final ShortUrl shortUrl; 12 | private final TimeFrame timeFrame; 13 | private final Map> counters = new HashMap>(); 14 | 15 | public ShortUrlStatistics(ShortUrl shortUrl, TimeFrame timeFrame) { 16 | this.shortUrl = shortUrl; 17 | this.timeFrame = timeFrame; 18 | } 19 | 20 | public ShortUrl getShortUrl() { 21 | return shortUrl; 22 | } 23 | 24 | public TimeFrame getTimeFrame() { 25 | return timeFrame; 26 | } 27 | 28 | public NavigableSet getCounters(String name) { 29 | return counters.get(name); 30 | } 31 | 32 | public void addCounters(String name, NavigableSet counters) { 33 | this.counters.put(name, counters); 34 | } 35 | } -------------------------------------------------------------------------------- /hush/src/main/java/com/hbasebook/hush/model/StatisticsCategory.java: -------------------------------------------------------------------------------- 1 | package com.hbasebook.hush.model; 2 | 3 | /** 4 | * All possible statistics saved in columns in the table. 5 | */ 6 | public enum StatisticsCategory { 7 | CLICK("cl"), COUNTRY("co"); 8 | 9 | private final String postfix; 10 | 11 | StatisticsCategory(String postfix) { 12 | this.postfix = postfix; 13 | } 14 | 15 | public String getPostfix() { 16 | return postfix; 17 | } 18 | 19 | public byte getCode() { 20 | return (byte) (ordinal() + 1); 21 | } 22 | 23 | @Override 24 | public String toString() { 25 | return postfix; 26 | } 27 | 28 | public static StatisticsCategory forPostfix(String postfix) { 29 | for (StatisticsCategory c : StatisticsCategory.values()) { 30 | if (c.postfix.equals(postfix)) { 31 | return c; 32 | } 33 | } 34 | return null; 35 | } 36 | 37 | public static StatisticsCategory forCode(int code) { 38 | for (StatisticsCategory c : StatisticsCategory.values()) { 39 | if (c.ordinal() + 1 == code) { 40 | return c; 41 | } 42 | } 43 | return null; 44 | } 45 | } -------------------------------------------------------------------------------- /hush/src/main/java/com/hbasebook/hush/model/TimeFrame.java: -------------------------------------------------------------------------------- 1 | package com.hbasebook.hush.model; 2 | 3 | /** 4 | * Time frame for statistics. 5 | */ 6 | public enum TimeFrame { 7 | DAY, WEEK, MONTH 8 | } -------------------------------------------------------------------------------- /hush/src/main/java/com/hbasebook/hush/model/User.java: -------------------------------------------------------------------------------- 1 | package com.hbasebook.hush.model; 2 | 3 | import java.util.HashSet; 4 | import java.util.Set; 5 | 6 | import com.hbasebook.hush.UserManager; 7 | 8 | public class User { 9 | public static final String USER_ROLE = "user"; 10 | public static final String ADMIN_ROLE = "admin"; 11 | 12 | private final String username; 13 | private final String firstName; 14 | private final String lastName; 15 | private final String email; 16 | private final String credentials; 17 | private final String roles; 18 | private transient Set roleSet = null; 19 | 20 | public User(String username, String firstName, String lastName, 21 | String email, String credentials, String roles) { 22 | super(); 23 | this.username = username; 24 | this.firstName = firstName; 25 | this.lastName = lastName; 26 | this.email = email; 27 | this.credentials = credentials; 28 | this.roles = roles; 29 | } 30 | 31 | public String getUsername() { 32 | return username; 33 | } 34 | 35 | public String getEmail() { 36 | return email; 37 | } 38 | 39 | public String getFirstName() { 40 | return firstName; 41 | } 42 | 43 | public String getLastName() { 44 | return lastName; 45 | } 46 | 47 | public String getCredentials() { 48 | return credentials; 49 | } 50 | 51 | public String getRoles() { 52 | return roles; 53 | } 54 | 55 | public Set getRoleSet() { 56 | if (roleSet == null) { 57 | roleSet = new HashSet(); 58 | if (roles != null) { 59 | for (String role : roles.split(",")) { 60 | roleSet.add(role.trim()); 61 | } 62 | } 63 | } 64 | return roleSet; 65 | } 66 | 67 | public boolean isAdmin() { 68 | return getRoleSet().contains(ADMIN_ROLE); 69 | } 70 | 71 | public static boolean isAnonymous(String username) { 72 | return UserManager.isAnonymous(username); 73 | } 74 | 75 | public static String displayName(String username) { 76 | if (isAnonymous(username)) { 77 | return "anonymous"; 78 | } else { 79 | return username; 80 | } 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /hush/src/main/java/com/hbasebook/hush/servlet/RequestInfo.java: -------------------------------------------------------------------------------- 1 | package com.hbasebook.hush.servlet; 2 | 3 | import javax.servlet.http.HttpServletRequest; 4 | import java.util.HashMap; 5 | import java.util.Map; 6 | 7 | /** 8 | * Stores the request details needed for statistics. 9 | */ 10 | public class RequestInfo { 11 | 12 | public enum InfoName { RemoteAddr, UserAgent } 13 | 14 | private final Map info; 15 | 16 | public RequestInfo() { 17 | info = new HashMap(); 18 | } 19 | 20 | public RequestInfo(Map info) { 21 | this.info = info; 22 | } 23 | 24 | public RequestInfo(HttpServletRequest request) { 25 | info = new HashMap(); 26 | if (request != null) { 27 | info.put(InfoName.RemoteAddr, request.getRemoteAddr()); 28 | } 29 | } 30 | 31 | public String get(InfoName name) { 32 | return info.get(name); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /hush/src/main/java/com/hbasebook/hush/servlet/security/HBaseLoginService.java: -------------------------------------------------------------------------------- 1 | package com.hbasebook.hush.servlet.security; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.commons.logging.Log; 6 | import org.apache.commons.logging.LogFactory; 7 | import org.eclipse.jetty.util.security.Credential; 8 | import org.eclipse.jetty.security.MappedLoginService; 9 | import org.eclipse.jetty.server.UserIdentity; 10 | 11 | import com.hbasebook.hush.ResourceManager; 12 | import com.hbasebook.hush.UserManager; 13 | import com.hbasebook.hush.model.User; 14 | 15 | /** 16 | * Implements the Jetty LoginService instance required for server 17 | * based authentication. 18 | */ 19 | public class HBaseLoginService extends MappedLoginService { 20 | private final Log LOG = LogFactory.getLog(HBaseLoginService.class); 21 | 22 | public HBaseLoginService(String name) { 23 | super(); 24 | setName(name); 25 | } 26 | 27 | @Override 28 | protected UserIdentity loadUser(String username) { 29 | try { 30 | UserManager manager = ResourceManager.getInstance().getUserManager(); 31 | User user = manager.getUser(username); 32 | String roleString = user.getRoles(); 33 | String[] roles = roleString == null ? null : roleString.split(","); 34 | return putUser(username, Credential.getCredential(user.getCredentials()), 35 | roles); 36 | } catch (Exception e) { 37 | LOG.error(String.format("Unable to get user '%s'", username), e); 38 | return null; 39 | } 40 | } 41 | 42 | @Override 43 | protected void loadUsers() throws IOException { 44 | UserManager manager = ResourceManager.getInstance().getUserManager(); 45 | for (User user : manager.getUsers()) { 46 | String roleString = user.getRoles(); 47 | String[] roles = roleString == null ? null : roleString.split(","); 48 | putUser(user.getCredentials(), Credential.getCredential( 49 | user.getCredentials()), roles); 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /hush/src/main/java/com/hbasebook/hush/table/HushTable.java: -------------------------------------------------------------------------------- 1 | package com.hbasebook.hush.table; 2 | 3 | import org.apache.hadoop.hbase.TableName; 4 | import org.apache.hadoop.hbase.util.Bytes; 5 | 6 | public class HushTable { 7 | public static final TableName NAME = TableName.valueOf("hush", "hush"); 8 | public static final byte[] COUNTERS_FAMILY = Bytes.toBytes("cnt"); 9 | public static final byte[] SHORT_ID = Bytes.toBytes("sid"); 10 | public static final byte[] ANONYMOUS_USER_ID = Bytes.toBytes("auid"); 11 | public static final byte[] GLOBAL_ROW_KEY = Bytes.toBytes("global"); 12 | } 13 | -------------------------------------------------------------------------------- /hush/src/main/java/com/hbasebook/hush/table/LongDomainTable.java: -------------------------------------------------------------------------------- 1 | package com.hbasebook.hush.table; 2 | 3 | import org.apache.hadoop.hbase.TableName; 4 | import org.apache.hadoop.hbase.util.Bytes; 5 | 6 | public class LongDomainTable { 7 | public static final TableName NAME = TableName.valueOf("hush", "ldom"); 8 | public static final byte[] DATA_FAMILY = Bytes.toBytes("data"); 9 | public static final byte[] SHORT_DOMAIN = Bytes.toBytes("sdom"); 10 | } 11 | -------------------------------------------------------------------------------- /hush/src/main/java/com/hbasebook/hush/table/LongUrlTable.java: -------------------------------------------------------------------------------- 1 | package com.hbasebook.hush.table; 2 | 3 | import org.apache.hadoop.hbase.TableName; 4 | import org.apache.hadoop.hbase.util.Bytes; 5 | 6 | public class LongUrlTable { 7 | public static final TableName NAME = TableName.valueOf("hush", "url"); 8 | public static final byte[] DATA_FAMILY = Bytes.toBytes("data"); 9 | public static final byte[] URL = Bytes.toBytes("url"); 10 | public static final byte[] SHORT_ID = Bytes.toBytes("sid"); 11 | } 12 | -------------------------------------------------------------------------------- /hush/src/main/java/com/hbasebook/hush/table/ShortDomainTable.java: -------------------------------------------------------------------------------- 1 | package com.hbasebook.hush.table; 2 | 3 | import org.apache.hadoop.hbase.TableName; 4 | import org.apache.hadoop.hbase.util.Bytes; 5 | 6 | public class ShortDomainTable { 7 | public static final TableName NAME = TableName.valueOf("hush", "sdom"); 8 | public static final byte[] DOMAINS_FAMILY = Bytes.toBytes("domains"); 9 | } 10 | -------------------------------------------------------------------------------- /hush/src/main/java/com/hbasebook/hush/table/ShortUrlTable.java: -------------------------------------------------------------------------------- 1 | package com.hbasebook.hush.table; 2 | 3 | import org.apache.hadoop.hbase.TableName; 4 | import org.apache.hadoop.hbase.util.Bytes; 5 | 6 | public class ShortUrlTable { 7 | public static final TableName NAME = TableName.valueOf("hush", "surl"); 8 | public static final byte[] DATA_FAMILY = Bytes.toBytes("data"); 9 | public static final byte[] DAILY_FAMILY = Bytes.toBytes("std"); 10 | public static final byte[] WEEKLY_FAMILY = Bytes.toBytes("stw"); 11 | public static final byte[] MONTHLY_FAMILY = Bytes.toBytes("stm"); 12 | public static final byte[] URL = Bytes.toBytes("url"); 13 | public static final byte[] SHORT_DOMAIN = Bytes.toBytes("sdom"); 14 | public static final byte[] REF_SHORT_ID = Bytes.toBytes("ref"); 15 | public static final byte[] USER_ID = Bytes.toBytes("uid"); 16 | public static final byte[] CLICKS = Bytes.toBytes("clk"); 17 | } 18 | -------------------------------------------------------------------------------- /hush/src/main/java/com/hbasebook/hush/table/UserShortUrlTable.java: -------------------------------------------------------------------------------- 1 | package com.hbasebook.hush.table; 2 | 3 | import org.apache.hadoop.hbase.TableName; 4 | import org.apache.hadoop.hbase.util.Bytes; 5 | 6 | public class UserShortUrlTable { 7 | public static final TableName NAME = TableName.valueOf("hush", "user-surl"); 8 | public static final byte[] DATA_FAMILY = Bytes.toBytes("data"); 9 | public static final byte[] TIMESTAMP = Bytes.toBytes("ts"); 10 | } 11 | -------------------------------------------------------------------------------- /hush/src/main/java/com/hbasebook/hush/table/UserTable.java: -------------------------------------------------------------------------------- 1 | package com.hbasebook.hush.table; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.hbase.TableName; 6 | import org.apache.hadoop.hbase.util.Bytes; 7 | 8 | public class UserTable { 9 | public static final TableName NAME = TableName.valueOf("hush", "user"); 10 | public static final byte[] DATA_FAMILY = Bytes.toBytes("data"); 11 | 12 | public static final byte[] CREDENTIALS = Bytes.toBytes("credentials"); 13 | public static final byte[] ROLES = Bytes.toBytes("roles"); 14 | 15 | public static final byte[] ADMIN_ROLE = Bytes.toBytes("admin"); 16 | public static final byte[] USER_ROLE = Bytes.toBytes("user"); 17 | public static final byte[] ADMIN_ROLES = Bytes.add(ADMIN_ROLE, 18 | Bytes.toBytes(","), USER_ROLE); 19 | 20 | public static final byte[] FIRSTNAME = Bytes.toBytes("firstname"); 21 | public static final byte[] LASTNAME = Bytes.toBytes("lastname"); 22 | public static final byte[] EMAIL = Bytes.toBytes("email"); 23 | 24 | public void addUser(String username) throws IOException { 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /hush/src/main/java/com/maxmind/geoip/Country.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Country.java 3 | * 4 | * Copyright (C) 2003 MaxMind LLC. All Rights Reserved. 5 | * 6 | * This library is free software; you can redistribute it and/or 7 | * modify it under the terms of the GNU Lesser General Public 8 | * License as published by the Free Software Foundation; either 9 | * version 2 of the License, or (at your option) any later version. 10 | * 11 | * This library is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Lesser General Public 17 | * License along with this library; if not, write to the Free Software 18 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 19 | */ 20 | 21 | package com.maxmind.geoip; 22 | 23 | /** 24 | * Represents a country. 25 | * 26 | * @author Matt Tucker 27 | */ 28 | public class Country { 29 | 30 | private String code; 31 | private String name; 32 | 33 | /** 34 | * Creates a new Country. 35 | * 36 | * @param code the country code. 37 | * @param name the country name. 38 | */ 39 | public Country(String code, String name) { 40 | this.code = code; 41 | this.name = name; 42 | } 43 | 44 | /** 45 | * Returns the ISO two-letter country code of this country. 46 | * 47 | * @return the country code. 48 | */ 49 | public String getCode() { 50 | return code; 51 | } 52 | 53 | /** 54 | * Returns the name of this country. 55 | * 56 | * @return the country name. 57 | */ 58 | public String getName() { 59 | return name; 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /hush/src/main/java/com/maxmind/geoip/Location.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Location.java 3 | * 4 | * Copyright (C) 2004 MaxMind LLC. All Rights Reserved. 5 | * 6 | * This library is free software; you can redistribute it and/or 7 | * modify it under the terms of the GNU General Lesser Public 8 | * License as published by the Free Software Foundation; either 9 | * version 2 of the License, or (at your option) any later version. 10 | * 11 | * This library is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Lesser General Public 17 | * License along with this library; if not, write to the Free Software 18 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 19 | */ 20 | 21 | package com.maxmind.geoip; 22 | 23 | public class Location { 24 | public String countryCode; 25 | public String countryName; 26 | public String region; 27 | public String city; 28 | public String postalCode; 29 | public float latitude; 30 | public float longitude; 31 | public int dma_code; 32 | public int area_code; 33 | public int metro_code; 34 | 35 | private final static double EARTH_DIAMETER = 2 * 6378.2; 36 | private final static double PI = 3.14159265; 37 | private final static double RAD_CONVERT = PI / 180; 38 | 39 | public double distance (Location loc) { 40 | double delta_lat, delta_lon; 41 | double temp; 42 | 43 | float lat1 = latitude; 44 | float lon1 = longitude; 45 | float lat2 = loc.latitude; 46 | float lon2 = loc.longitude; 47 | 48 | // convert degrees to radians 49 | lat1 *= RAD_CONVERT; 50 | lat2 *= RAD_CONVERT; 51 | 52 | // find the deltas 53 | delta_lat = lat2 - lat1; 54 | delta_lon = (lon2 - lon1) * RAD_CONVERT; 55 | 56 | // Find the great circle distance 57 | temp = Math.pow(Math.sin(delta_lat/2),2) + Math.cos(lat1) * Math.cos(lat2) * Math.pow(Math.sin(delta_lon/2),2); 58 | return EARTH_DIAMETER * Math.atan2(Math.sqrt(temp),Math.sqrt(1-temp)); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /hush/src/main/java/com/maxmind/geoip/Region.java: -------------------------------------------------------------------------------- 1 | package com.maxmind.geoip; 2 | 3 | public class Region{ 4 | public String countryCode; 5 | public String countryName; 6 | public String region; 7 | } 8 | 9 | -------------------------------------------------------------------------------- /hush/src/main/resources/GeoIP.dat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/larsgeorge/hbase-book/2ecb9a13820e7effb12c4ae1cb998eddc1414e46/hush/src/main/resources/GeoIP.dat -------------------------------------------------------------------------------- /hush/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO, stdout 2 | 3 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 4 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 5 | 6 | # Pattern to output the caller's file name and line number. 7 | log4j.appender.stdout.layout.ConversionPattern=%5p [%t] (%F:%L) - %m%n 8 | -------------------------------------------------------------------------------- /hush/src/main/webapp/admin/index.jsp: -------------------------------------------------------------------------------- 1 | <% 2 | response.sendRedirect("/admin/domains.jsp"); 3 | return ; 4 | %> -------------------------------------------------------------------------------- /hush/src/main/webapp/admin/users.jsp: -------------------------------------------------------------------------------- 1 | <%@ page contentType="text/html;charset=UTF-8"%> 2 | <%@ page import="java.util.List"%> 3 | <%@ page import="com.hbasebook.hush.model.User"%> 4 | <%@ page import="com.hbasebook.hush.HushUtil"%> 5 | <%@ page import="com.hbasebook.hush.ResourceManager"%> 6 | <%@ page import="com.hbasebook.hush.UserManager"%> 7 | <% 8 | UserManager um = ResourceManager.getInstance().getUserManager(); 9 | List list = um.getUsers(); 10 | %> 11 | 13 | 14 | 15 | Users 16 | 17 | 18 | 19 |
20 | 21 |
22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | <% 35 | for (User user : list) { 36 | String fullName = HushUtil.fixNull(user.getFirstName()) + ' ' 37 | + HushUtil.fixNull(user.getLastName()); 38 | %> 39 | 40 | 42 | 43 | 44 | 45 | 46 | <% 47 | } 48 | %> 49 | 50 |
UsernameNameE-mailRoles
41 | <%= user.getUsername() %><%= fullName %><%=HushUtil.fixNull(user.getRoles())%>
51 |
52 |
53 | 54 | 55 | -------------------------------------------------------------------------------- /hush/src/main/webapp/error.jsp: -------------------------------------------------------------------------------- 1 | <%@ page contentType="text/html;charset=UTF-8" %> 2 | 4 | 5 | 6 | Fail 7 | 8 | 9 | 10 |
11 |

Shush! There was a fail.

12 | 13 | Go home 14 |
15 | 16 | -------------------------------------------------------------------------------- /hush/src/main/webapp/include/adminMenu.jsp: -------------------------------------------------------------------------------- 1 | <%@ page import="java.util.List" %> 2 | <%@ page import="java.util.ArrayList" %> 3 |
4 |
    5 | <% 6 | String uri = request.getRequestURI(); 7 | List menu = new ArrayList(); 8 | menu.add (new String [] { "domains","/admin/domains.jsp" }); 9 | menu.add (new String [] { "users","/admin/users.jsp" }); 10 | 11 | for (String [] item : menu) { 12 | if (uri.startsWith(item[1])) { 13 | %> 14 |
  • <%= item[0] %>
    • 15 | <% 16 | } else { 17 | %> 18 |
  • <%= item[0] %>
    • 19 | <% 20 | } 21 | } 22 | %> 23 |
24 |
25 | -------------------------------------------------------------------------------- /hush/src/main/webapp/include/error.jsp: -------------------------------------------------------------------------------- 1 | <% 2 | String error = request.getParameter("error"); 3 | if (error == null) { 4 | error = (String)request.getAttribute("error"); 5 | } 6 | if (error != null) { 7 | %> 8 |
9 |

<%=error%>

10 |
11 | <% 12 | } 13 | %> 14 | -------------------------------------------------------------------------------- /hush/src/main/webapp/include/footer.jsp: -------------------------------------------------------------------------------- 1 | 4 | -------------------------------------------------------------------------------- /hush/src/main/webapp/include/header.jsp: -------------------------------------------------------------------------------- 1 | <%@ page import="java.security.Principal" %> 2 | <% 3 | Principal principal = request.getUserPrincipal(); 4 | %> 5 | 15 | -------------------------------------------------------------------------------- /hush/src/main/webapp/index.jsp: -------------------------------------------------------------------------------- 1 | <%@ page contentType="text/html;charset=UTF-8" %> 2 | <%@ page import="com.hbasebook.hush.ResourceManager" %> 3 | <%@ page import="com.hbasebook.hush.UrlManager" %> 4 | <%@ page import="com.hbasebook.hush.HushUtil" %> 5 | <%@ page import="com.hbasebook.hush.model.ShortUrl" %> 6 | <%@ page import="java.net.URL" %> 7 | <%@ page import="java.security.Principal" %> 8 | <%@ page import="java.net.MalformedURLException" %> 9 | <%@ page import="com.hbasebook.hush.servlet.RequestInfo" %> 10 | <% 11 | ShortUrl surl = null; 12 | String urlParam = request.getParameter("url"); 13 | if (urlParam != null && urlParam.length() > 0) { 14 | try { 15 | URL url = new URL(urlParam); 16 | UrlManager urlm = ResourceManager.getInstance().getUrlManager(); 17 | String username = HushUtil.getOrSetUsername(request, response); 18 | surl = urlm.shorten(url, username, new RequestInfo(request)); 19 | } catch (MalformedURLException e) { 20 | request.setAttribute("error", "Invalid URL."); 21 | } 22 | } 23 | %> 24 | 26 | 27 | 28 | Hush! 29 | 30 | 31 | 32 |
33 | 34 |
35 |

Welcome to the HBase URL Shortener

36 | 37 | 38 |
39 |

Shorten your URLs!

40 | 41 |
42 | 43 | 44 |
45 |
46 | <% if (surl != null) { 47 | String qrUrl = surl.toString() + ".q"; 48 | %> 49 |
50 |

Your new shortened URL is:

51 | 53 |
54 | <% } %> 55 | 56 |
57 |
58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /hush/src/main/webapp/login.jsp: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | Log in 6 | 7 | 8 | 9 |
10 |
11 |
12 |

Log in

13 | 14 |

New users sign up here.

15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 |
23 |
24 |
25 |
26 | 27 | -------------------------------------------------------------------------------- /hush/src/main/webapp/logout.jsp: -------------------------------------------------------------------------------- 1 | <% 2 | session.invalidate(); 3 | response.sendRedirect("/index.jsp"); 4 | %> 5 | --------------------------------------------------------------------------------