qlass, U handler) throws InstantiationException {
59 | return ClassProxy.create(qlass, new UntypedProxy(handler));
60 | }
61 |
62 | /**
63 | * Initialises the proxy.
64 | *
65 | * @param target Target handler for intercepted/proxied method calls.
66 | */
67 | public UntypedProxy(T target) {
68 | mTarget = target;
69 | }
70 |
71 | @Override
72 | public Object intercept(
73 | Object self,
74 | java.lang.reflect.Method method,
75 | Object[] args,
76 | MethodProxy proxy)
77 | throws Throwable {
78 | // Forwards the method call to the underlying handler, through reflection:
79 | try {
80 | return mTarget.getClass()
81 | .getMethod(method.getName(), method.getParameterTypes())
82 | .invoke(mTarget, args);
83 | } catch (InvocationTargetException ite) {
84 | throw ite.getCause();
85 | }
86 | }
87 | }
88 |
--------------------------------------------------------------------------------
/src/test/scala/org/kiji/testing/fakehtable/TestFakeHBase.scala:
--------------------------------------------------------------------------------
1 | /**
2 | * (c) Copyright 2012 WibiData, Inc.
3 | *
4 | * See the NOTICE file distributed with this work for additional
5 | * information regarding copyright ownership.
6 | *
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | */
19 |
20 | package org.kiji.testing.fakehtable
21 |
22 | import scala.collection.JavaConverters.asScalaBufferConverter
23 |
24 | import org.apache.commons.codec.binary.Hex
25 | import org.apache.hadoop.hbase.HTableDescriptor
26 | import org.apache.hadoop.hbase.HBaseConfiguration
27 | import org.apache.hadoop.hbase.client.HTable
28 | import org.junit.Assert
29 | import org.junit.Test
30 |
31 | /** Tests for the FakeHBase class. */
32 | class TestFakeHBase {
33 |
34 | /** Test the basic API of FakeHBase. */
35 | @Test
36 | def testFakeHBase(): Unit = {
37 | val hbase = new FakeHBase()
38 | val desc = new HTableDescriptor("table-name")
39 | hbase.Admin.createTable(desc)
40 |
41 | val tables = hbase.Admin.listTables()
42 | Assert.assertEquals(1, tables.length)
43 | Assert.assertEquals("table-name", tables(0).getNameAsString())
44 | }
45 |
46 | /** Test the fake implementation of HBaseAdmin.getTableRegions(). */
47 | @Test
48 | def testSimpleRegionSplit(): Unit = {
49 | val hbase = new FakeHBase()
50 | val desc = new HTableDescriptor("table-name")
51 | hbase.Admin.createTable(desc, null, null, numRegions = 2)
52 |
53 | val regions = hbase.Admin.getTableRegions("table-name".getBytes).asScala
54 | Assert.assertEquals(2, regions.size)
55 | assert(regions.head.getStartKey.isEmpty)
56 | assert(regions.last.getEndKey.isEmpty)
57 | for (i <- 0 until regions.size - 1) {
58 | Assert.assertEquals(
59 | regions(i).getEndKey.toSeq,
60 | regions(i + 1).getStartKey.toSeq)
61 | }
62 | Assert.assertEquals(
63 | "7fffffffffffffffffffffffffffffff",
64 | Hex.encodeHexString(regions(0).getEndKey))
65 | }
66 |
67 | /** Tests that FakeHTable instances appear as valid instances of HTable. */
68 | @Test
69 | def testFakeHTableAsInstanceOfHTable(): Unit = {
70 | val hbase = new FakeHBase()
71 | val desc = new HTableDescriptor("table")
72 | hbase.Admin.createTable(desc)
73 | val conf = HBaseConfiguration.create()
74 | val htable: HTable = hbase.InterfaceFactory.create(conf, "table").asInstanceOf[HTable]
75 | val locations = htable.getRegionLocations()
76 | Assert.assertEquals(1, locations.size)
77 | val location = htable.getRegionLocation("row key")
78 | Assert.assertEquals(locations.keySet.iterator.next, location.getRegionInfo)
79 | }
80 |
81 | @Test
82 | def testAdminFactory(): Unit = {
83 | val hbase = new FakeHBase()
84 | val conf = HBaseConfiguration.create()
85 | val admin = hbase.AdminFactory.create(conf)
86 | admin.close()
87 | }
88 | }
89 |
--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
21 | 4.0.0
22 |
23 | org.kiji.testing
24 | fake-hbase
25 | 0.3.0-SNAPSHOT
26 | jar
27 |
28 |
29 | org.kiji.pom
30 | root-pom
31 | 1.2.1-SNAPSHOT
32 |
33 |
34 | fake-hbase
35 |
36 | Fake HBase table implementation, for testing purposes.
37 |
38 | http://www.kiji.org
39 |
40 |
41 | 2.3.0-cdh5.0.3
42 | 0.96.1.1-cdh5.0.3
43 |
44 |
45 |
46 |
47 |
48 | net.alchim31.maven
49 | scala-maven-plugin
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 | org.scala-lang
58 | scala-library
59 | compile
60 |
61 |
62 |
63 | org.apache.hadoop
64 | hadoop-common
65 | ${hadoop.version}
66 | provided
67 |
68 |
69 |
70 | org.apache.hbase
71 | hbase-client
72 | ${hbase.version}
73 | provided
74 |
75 |
76 |
77 | org.apache.hadoop
78 | hadoop-core
79 |
80 |
81 |
82 |
83 |
84 |
85 | org.easymock
86 | easymock
87 | compile
88 |
89 |
90 |
91 |
92 | junit
93 | junit
94 | test
95 |
96 |
97 |
98 |
99 |
100 | kiji-repos
101 | kiji-repos
102 | https://repo.wibidata.com/artifactory/kiji
103 |
104 | false
105 |
106 |
107 | true
108 |
109 |
110 |
111 |
112 |
113 |
--------------------------------------------------------------------------------
/src/main/scala/org/kiji/testing/fakehtable/FakeHBase.scala:
--------------------------------------------------------------------------------
1 | /**
2 | * (c) Copyright 2012 WibiData, Inc.
3 | *
4 | * See the NOTICE file distributed with this work for additional
5 | * information regarding copyright ownership.
6 | *
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | */
19 |
20 | package org.kiji.testing.fakehtable
21 |
22 | import java.lang.{Integer => JInteger}
23 | import java.util.Arrays
24 | import java.util.{List => JList}
25 | import java.util.{TreeMap => JTreeMap}
26 |
27 | import scala.collection.JavaConverters.asScalaIteratorConverter
28 | import scala.collection.mutable.Buffer
29 | import scala.math.BigInt.int2bigInt
30 |
31 | import org.apache.hadoop.conf.Configuration
32 | import org.apache.hadoop.hbase.HColumnDescriptor
33 | import org.apache.hadoop.hbase.HRegionInfo
34 | import org.apache.hadoop.hbase.HTableDescriptor
35 | import org.apache.hadoop.hbase.TableExistsException
36 | import org.apache.hadoop.hbase.TableNotDisabledException
37 | import org.apache.hadoop.hbase.TableNotFoundException
38 | import org.apache.hadoop.hbase.client.HBaseAdmin
39 | import org.apache.hadoop.hbase.client.HConnection
40 | import org.apache.hadoop.hbase.client.HTable
41 | import org.apache.hadoop.hbase.client.HTableInterface
42 | import org.apache.hadoop.hbase.util.Bytes
43 | import org.apache.hadoop.hbase.util.Pair
44 | import org.kiji.schema.impl.HBaseAdminFactory
45 | import org.kiji.schema.impl.HBaseInterface
46 |
47 | /**
48 | * Fake HBase instance, as a collection of fake HTable instances.
49 | *
50 | * FakeHBase/FakeHConnection act as factories for FakeHTable.
51 | * Conceptually, there is a single FakeHConnection per FakeHBase.
52 | */
53 | class FakeHBase
54 | extends HBaseInterface
55 | with FakeTypes {
56 |
57 | /**
58 | * Controls whether to automatically create unknown tables or throw a TableNotFoundException.
59 | *
60 | * TODO: Drop this feature, it seems like a bad idea as unknown tables have unspecified
61 | * properties (eg. max-versions, TTL, etc).
62 | */
63 | private var createUnknownTable: Boolean = false
64 |
65 | /** Default HConnection to connect to this HBase instance and the HTables it contains. */
66 | private val mFakeHConnection: FakeHConnection = new FakeHConnection(this)
67 | private val mHConnection: HConnection =
68 | UntypedProxy.create(classOf[HConnection], mFakeHConnection)
69 |
70 | /** Map of the FakeHTable in this FakeHBase instance, keyed on HTable name. */
71 | private[fakehtable] val tableMap = new JTreeMap[Bytes, FakeHTable](Bytes.BYTES_COMPARATOR)
72 |
73 | /**
74 | * Enables or disables the «create unknown table» feature.
75 | *
76 | * @param createUnknownTableFlag Whether unknown tables should be implicitly created.
77 | * When disabled, TableNotFoundException is raised.
78 | */
79 | def setCreateUnknownTable(createUnknownTableFlag: Boolean): Unit = {
80 | synchronized {
81 | this.createUnknownTable = createUnknownTableFlag
82 | }
83 | }
84 |
85 | // -----------------------------------------------------------------------------------------------
86 |
87 | /** Factory for HTableInterface instances. */
88 | object InterfaceFactory
89 | extends org.kiji.schema.impl.HTableInterfaceFactory
90 | with org.apache.hadoop.hbase.client.HTableInterfaceFactory {
91 |
92 | override def create(conf: Configuration, tableName: String): HTableInterface = {
93 | val tableNameBytes = Bytes.toBytes(tableName)
94 | synchronized {
95 | var table = tableMap.get(tableNameBytes)
96 | if (table == null) {
97 | if (!createUnknownTable) {
98 | throw new TableNotFoundException(tableName)
99 | }
100 | val desc = new HTableDescriptor(tableName)
101 | table = new FakeHTable(
102 | name = tableName,
103 | conf = conf,
104 | desc = desc,
105 | hconnection = mFakeHConnection
106 | )
107 | tableMap.put(tableNameBytes, table)
108 | }
109 | return UntypedProxy.create(classOf[HTable], table)
110 | }
111 | }
112 |
113 | override def createHTableInterface(
114 | conf: Configuration,
115 | tableName: Bytes
116 | ): HTableInterface = {
117 | return create(tableName = Bytes.toString(tableName), conf = conf)
118 | }
119 |
120 | override def releaseHTableInterface(table: HTableInterface): Unit = {
121 | // Do nothing
122 | }
123 | }
124 |
125 | override def getHTableFactory(): org.kiji.schema.impl.HTableInterfaceFactory = InterfaceFactory
126 |
127 | // -----------------------------------------------------------------------------------------------
128 |
129 | object Admin extends HBaseAdminCore with HBaseAdminConversionHelpers {
130 | def addColumn(tableName: Bytes, column: HColumnDescriptor): Unit = {
131 | // TODO(taton) Implement metadata
132 | // For now, do nothing
133 | }
134 |
135 | def createTable(desc: HTableDescriptor, split: Array[Bytes]): Unit = {
136 | synchronized {
137 | if (tableMap.containsKey(desc.getName)) {
138 | throw new TableExistsException(desc.getNameAsString)
139 | }
140 | val table = new FakeHTable(
141 | name = desc.getNameAsString,
142 | desc = desc,
143 | hconnection = mFakeHConnection
144 | )
145 | Arrays.sort(split, Bytes.BYTES_COMPARATOR)
146 | table.setSplit(split)
147 | tableMap.put(desc.getName, table)
148 | }
149 | }
150 |
151 | def createTable(
152 | desc: HTableDescriptor,
153 | startKey: Bytes,
154 | endKey: Bytes,
155 | numRegions: Int
156 | ): Unit = {
157 | // TODO Handle startKey/endKey
158 | val split = Buffer[Bytes]()
159 | val min = 0
160 | val max: BigInt = (BigInt(1) << 128) - 1
161 | for (n <- 1 until numRegions) {
162 | val boundary: Bytes = MD5Space(n, numRegions)
163 | split.append(boundary)
164 | }
165 | createTable(desc = desc, split = split.toArray)
166 | }
167 |
168 | def deleteColumn(tableName: Bytes, columnName: Bytes): Unit = {
169 | // TODO(taton) Implement metadata
170 | // For now, do nothing
171 | }
172 |
173 | def deleteTable(tableName: Bytes): Unit = {
174 | synchronized {
175 | val table = tableMap.get(tableName)
176 | if (table == null) {
177 | throw new TableNotFoundException(Bytes.toStringBinary(tableName))
178 | }
179 | if (table.enabled) {
180 | throw new TableNotDisabledException(tableName)
181 | }
182 | tableMap.remove(tableName)
183 | }
184 | }
185 |
186 | def disableTable(tableName: Bytes): Unit = {
187 | synchronized {
188 | val table = tableMap.get(tableName)
189 | if (table == null) {
190 | throw new TableNotFoundException(Bytes.toStringBinary(tableName))
191 | }
192 | table.enabled = false
193 | }
194 | }
195 |
196 | def enableTable(tableName: Bytes): Unit = {
197 | synchronized {
198 | val table = tableMap.get(tableName)
199 | if (table == null) {
200 | throw new TableNotFoundException(Bytes.toStringBinary(tableName))
201 | }
202 | table.enabled = true
203 | }
204 | }
205 |
206 | def flush(tableName: Bytes): Unit = {
207 | // Nothing to do
208 | }
209 |
210 | def getTableRegions(tableName: Bytes): JList[HRegionInfo] = {
211 | synchronized {
212 | val table = tableMap.get(tableName)
213 | if (table == null) {
214 | throw new TableNotFoundException(Bytes.toStringBinary(tableName))
215 | }
216 | return table.getRegions()
217 | }
218 | }
219 |
220 | def isTableAvailable(tableName: Bytes): Boolean = {
221 | return isTableEnabled(tableName)
222 | }
223 |
224 | def isTableEnabled(tableName: Bytes): Boolean = {
225 | synchronized {
226 | val table = tableMap.get(tableName)
227 | if (table == null) {
228 | throw new TableNotFoundException(Bytes.toStringBinary(tableName))
229 | }
230 | return table.enabled
231 | }
232 | }
233 |
234 | def listTables(): Array[HTableDescriptor] = {
235 | synchronized {
236 | return tableMap.values.iterator.asScala
237 | .map { table => table.getTableDescriptor }
238 | .toArray
239 | }
240 | }
241 |
242 | def modifyColumn(tableName: Bytes, column: HColumnDescriptor): Unit = {
243 | // TODO(taton) Implement metadata
244 | }
245 |
246 | def modifyTable(tableName: Bytes, desc: HTableDescriptor): Unit = {
247 | // TODO(taton) Implement metadata
248 | }
249 |
250 | def getAlterStatus(tableName: Bytes): Pair[JInteger, JInteger] = {
251 | return new Pair(0, getTableRegions(tableName).size)
252 | }
253 |
254 | def tableExists(tableName: Bytes): Boolean = {
255 | synchronized {
256 | return tableMap.containsKey(tableName)
257 | }
258 | }
259 | }
260 |
261 | // -----------------------------------------------------------------------------------------------
262 |
263 | /** Factory for HBaseAdmin instances. */
264 | object AdminFactory extends HBaseAdminFactory {
265 | /** Creates a new HBaseAdmin for this HBase instance. */
266 | override def create(conf: Configuration): HBaseAdmin = {
267 | return UntypedProxy.create(classOf[HBaseAdmin], Admin)
268 | }
269 | }
270 |
271 | override def getAdminFactory(): HBaseAdminFactory = {
272 | AdminFactory
273 | }
274 |
275 | /**
276 | * Returns an HConnection for this fake HBase instance.
277 | *
278 | * @returns an HConnection for this fake HBase instance.
279 | */
280 | def getHConnection(): HConnection = {
281 | return mHConnection
282 | }
283 | }
284 |
--------------------------------------------------------------------------------
/src/main/scala/org/kiji/testing/fakehtable/HBaseAdminInterface.scala:
--------------------------------------------------------------------------------
1 | /**
2 | * (c) Copyright 2012 WibiData, Inc.
3 | *
4 | * See the NOTICE file distributed with this work for additional
5 | * information regarding copyright ownership.
6 | *
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | */
19 |
20 | package org.kiji.testing.fakehtable
21 |
22 | import java.io.Closeable
23 | import java.lang.{Integer => JInteger}
24 | import java.util.regex.Pattern
25 | import java.util.Arrays
26 | import java.util.{List => JList}
27 |
28 | import scala.collection.mutable
29 | import scala.collection.JavaConverters.asScalaBufferConverter
30 |
31 | import org.apache.hadoop.hbase.util.Bytes.toBytes
32 | import org.apache.hadoop.hbase.util.Bytes
33 | import org.apache.hadoop.hbase.util.Pair
34 | import org.apache.hadoop.hbase.HColumnDescriptor
35 | import org.apache.hadoop.hbase.HRegionInfo
36 | import org.apache.hadoop.hbase.HTableDescriptor
37 | import org.apache.hadoop.hbase.TableNotFoundException
38 |
39 |
40 | /** Core HBaseAdmin interface. */
41 | trait HBaseAdminCore
42 | extends Closeable {
43 |
44 | type Bytes = Array[Byte]
45 |
46 | def addColumn(tableName: Bytes, column: HColumnDescriptor): Unit
47 |
48 | def createTable(desc: HTableDescriptor, split: Array[Bytes]): Unit
49 | def createTable(
50 | desc: HTableDescriptor,
51 | startKey: Bytes,
52 | endKey: Bytes,
53 | numRegions: Int
54 | ): Unit
55 |
56 | def deleteColumn(tableName: Bytes, columnName: Bytes): Unit
57 |
58 | def deleteTable(tableName: Bytes): Unit
59 |
60 | def disableTable(tableName: Bytes): Unit
61 |
62 | def enableTable(tableName: Bytes): Unit
63 |
64 | def flush(tableName: Bytes): Unit
65 |
66 | def getTableRegions(tableName: Bytes): JList[HRegionInfo]
67 |
68 | def isTableAvailable(tableName: Bytes): Boolean
69 |
70 | def isTableEnabled(tableName: Bytes): Boolean
71 |
72 | def listTables(): Array[HTableDescriptor]
73 |
74 | def modifyColumn(tableName: Bytes, column: HColumnDescriptor): Unit
75 |
76 | def modifyTable(tableName: Bytes, desc: HTableDescriptor): Unit
77 |
78 | def getAlterStatus(tableName: Bytes): Pair[JInteger, JInteger]
79 |
80 | def tableExists(tableName: Bytes): Boolean
81 | }
82 |
83 |
84 | /**
85 | * Pure interface for compatibility with the concrete Java class HBaseAdmin.
86 | *
87 | * Extends HBaseAdminCore with helpers to convert between Bytes and String.
88 | */
89 | trait HBaseAdminInterface
90 | extends HBaseAdminCore {
91 |
92 | def addColumn(tableName: String, column: HColumnDescriptor): Unit
93 | def addColumn(tableName: Bytes, column: HColumnDescriptor): Unit
94 |
95 | def createTable(desc: HTableDescriptor): Unit
96 | def createTable(desc: HTableDescriptor, split: Array[Bytes]): Unit
97 | def createTable(
98 | desc: HTableDescriptor,
99 | startKey: Bytes, endKey: Bytes, numRegions: Int
100 | ): Unit
101 | def createTableAsync(desc: HTableDescriptor, split: Array[Bytes]): Unit
102 |
103 | def deleteColumn(tableName: String, columnName: String): Unit
104 | def deleteColumn(tableName: Bytes, columnName: Bytes): Unit
105 |
106 | def deleteTable(tableName: String): Unit
107 | def deleteTable(tableName: Bytes): Unit
108 | def deleteTables(regex: String): Unit
109 | def deleteTables(pattern: Pattern): Unit
110 |
111 | def disableTable(tableName: String): Unit
112 | def disableTable(tableName: Bytes): Unit
113 | def disableTableAsync(tableName: String): Unit
114 | def disableTableAsync(tableName: Bytes): Unit
115 | def disableTables(regex: String): Unit
116 | def disableTables(pattern: Pattern): Unit
117 |
118 | def enableTable(tableName: String): Unit
119 | def enableTable(tableName: Bytes): Unit
120 | def enableTableAsync(tableName: String): Unit
121 | def enableTableAsync(tableName: Bytes): Unit
122 | def enableTables(regex: String): Unit
123 | def enableTables(pattern: Pattern): Unit
124 |
125 | def flush(tableName: String): Unit
126 | def flush(tableName: Bytes): Unit
127 |
128 | // Similar to listTables()
129 | def getTableDescriptor(tableName: Bytes): HTableDescriptor
130 | def getTableDescriptors(tableNames: JList[String]): Array[HTableDescriptor]
131 |
132 | def getTableRegions(tableName: Bytes): JList[HRegionInfo]
133 |
134 | def isTableAvailable(tableName: String): Boolean
135 | def isTableAvailable(tableName: Bytes): Boolean
136 |
137 | def isTableDisabled(tableName: String): Boolean
138 | def isTableDisabled(tableName: Bytes): Boolean
139 |
140 | def isTableEnabled(tableName: String): Boolean
141 | def isTableEnabled(tableName: Bytes): Boolean
142 |
143 | def listTables(): Array[HTableDescriptor]
144 | def listTables(regex: String): Array[HTableDescriptor]
145 | def listTables(pattern: Pattern): Array[HTableDescriptor]
146 |
147 | def modifyColumn(tableName: String, column: HColumnDescriptor): Unit
148 | def modifyColumn(tableName: Bytes, column: HColumnDescriptor): Unit
149 |
150 | def modifyTable(tableName: Bytes, desc: HTableDescriptor): Unit
151 | def getAlterStatus(tableName: Bytes): Pair[JInteger, JInteger]
152 |
153 | def tableExists(tableName: String): Boolean
154 | def tableExists(tableName: Bytes): Boolean
155 | }
156 |
157 |
158 | /** Implements conversion helpers (Bytes/String, Regex/Pattern, etc). */
159 | trait HBaseAdminConversionHelpers extends HBaseAdminInterface {
160 | override def addColumn(tableName: String, column: HColumnDescriptor): Unit = {
161 | addColumn(toBytes(tableName), column)
162 | }
163 |
164 | override def close(): Unit = {
165 | // Do nothing
166 | }
167 |
168 | override def createTable(desc: HTableDescriptor): Unit = {
169 | createTable(desc, split = Array())
170 | }
171 |
172 | override def createTableAsync(desc: HTableDescriptor, split: Array[Bytes]): Unit = {
173 | createTable(desc, split)
174 | }
175 |
176 | override def deleteColumn(tableName: String, columnName: String): Unit = {
177 | deleteColumn(toBytes(tableName), toBytes(columnName))
178 | }
179 |
180 | override def deleteTable(tableName: String): Unit = {
181 | deleteTable(toBytes(tableName))
182 | }
183 |
184 | override def deleteTables(regex: String): Unit = {
185 | deleteTables(Pattern.compile(regex))
186 | }
187 | override def deleteTables(pattern: Pattern): Unit = {
188 | for (desc <- listTables()) {
189 | if (pattern.matcher(desc.getNameAsString).matches) {
190 | deleteTable(desc.getName)
191 | }
192 | }
193 | }
194 |
195 | override def disableTable(tableName: String): Unit = {
196 | disableTable(toBytes(tableName))
197 | }
198 | override def disableTableAsync(tableName: String): Unit = {
199 | disableTableAsync(toBytes(tableName))
200 | }
201 | override def disableTableAsync(tableName: Bytes): Unit = {
202 | disableTable(tableName)
203 | }
204 | override def disableTables(regex: String): Unit = {
205 | disableTables(Pattern.compile(regex))
206 | }
207 | override def disableTables(pattern: Pattern): Unit = {
208 | for (desc <- listTables()) {
209 | if (pattern.matcher(desc.getNameAsString).matches) {
210 | disableTable(desc.getName)
211 | }
212 | }
213 | }
214 |
215 | override def enableTable(tableName: String): Unit = {
216 | enableTable(toBytes(tableName))
217 | }
218 | override def enableTableAsync(tableName: String): Unit = {
219 | enableTableAsync(toBytes(tableName))
220 | }
221 | override def enableTableAsync(tableName: Bytes): Unit = {
222 | enableTable(tableName)
223 | }
224 | override def enableTables(regex: String): Unit = {
225 | enableTables(Pattern.compile(regex))
226 | }
227 | override def enableTables(pattern: Pattern): Unit = {
228 | for (desc <- listTables()) {
229 | if (pattern.matcher(desc.getNameAsString).matches) {
230 | enableTable(desc.getName)
231 | }
232 | }
233 | }
234 |
235 | override def flush(tableName: String): Unit = {
236 | flush(toBytes(tableName))
237 | }
238 |
239 | override def getTableDescriptor(tableName: Bytes): HTableDescriptor = {
240 | for (desc <- listTables()) {
241 | if (Arrays.equals(desc.getName, tableName)) {
242 | return desc
243 | }
244 | }
245 | throw new TableNotFoundException(Bytes.toStringBinary(tableName))
246 | }
247 | override def getTableDescriptors(tableNames: JList[String]): Array[HTableDescriptor] = {
248 | val descs = mutable.Buffer[HTableDescriptor]()
249 | val names = Set() ++ tableNames.asScala
250 | for (desc <- listTables()) {
251 | if (names.contains(desc.getNameAsString)) {
252 | descs += desc
253 | }
254 | }
255 | return descs.toArray
256 | }
257 |
258 | override def isTableAvailable(tableName: String): Boolean = {
259 | return isTableAvailable(toBytes(tableName))
260 | }
261 |
262 | override def isTableDisabled(tableName: String): Boolean = {
263 | return isTableDisabled(toBytes(tableName))
264 | }
265 | override def isTableDisabled(tableName: Bytes): Boolean = {
266 | return !isTableEnabled(tableName)
267 | }
268 |
269 | override def isTableEnabled(tableName: String): Boolean = {
270 | return isTableEnabled(toBytes(tableName))
271 | }
272 |
273 | override def listTables(regex: String): Array[HTableDescriptor] = {
274 | return listTables(Pattern.compile(regex))
275 | }
276 | override def listTables(pattern: Pattern): Array[HTableDescriptor] = {
277 | val descs = mutable.Buffer[HTableDescriptor]()
278 | for (desc <- listTables()) {
279 | if (pattern.matcher(desc.getNameAsString).matches) {
280 | descs += desc
281 | }
282 | }
283 | return descs.toArray
284 | }
285 |
286 | override def modifyColumn(tableName: String, column: HColumnDescriptor): Unit = {
287 | modifyColumn(toBytes(tableName), column)
288 | }
289 |
290 | override def tableExists(tableName: String): Boolean = {
291 | return tableExists(toBytes(tableName))
292 | }
293 | }
294 |
--------------------------------------------------------------------------------
/src/main/scala/org/kiji/testing/fakehtable/ProcessRow.scala:
--------------------------------------------------------------------------------
1 | /**
2 | * (c) Copyright 2013 WibiData, Inc.
3 | *
4 | * See the NOTICE file distributed with this work for additional
5 | * information regarding copyright ownership.
6 | *
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | */
19 |
20 | package org.kiji.testing.fakehtable
21 |
22 | import java.lang.{Long => JLong}
23 | import java.util.{ArrayList => JArrayList}
24 | import java.util.Arrays
25 | import java.util.{List => JList}
26 | import java.util.NavigableMap
27 | import java.util.NavigableSet
28 |
29 | import org.apache.hadoop.hbase.HColumnDescriptor
30 | import org.apache.hadoop.hbase.HConstants
31 | import org.apache.hadoop.hbase.Cell
32 | import org.apache.hadoop.hbase.CellUtil
33 | import org.apache.hadoop.hbase.KeyValue
34 | import org.apache.hadoop.hbase.client.Result
35 | import org.apache.hadoop.hbase.filter.Filter
36 | import org.apache.hadoop.hbase.io.TimeRange
37 | import org.kiji.testing.fakehtable.JNavigableMapWithAsScalaIterator.javaNavigableMapAsScalaIterator
38 | import org.slf4j.LoggerFactory
39 |
40 | /**
41 | * Utility object providing the logic to prepare HBase Result instances.
42 | *
43 | * FakeHTable is the only intended object of this class.
44 | */
45 | object ProcessRow extends FakeTypes {
46 | private final val Log = LoggerFactory.getLogger("ProcessRow")
47 |
48 | /** Comparator for KeyValue instances. */
49 | private final val KeyValueComparator = KeyValue.COMPARATOR
50 |
51 | /** Empty array of bytes. */
52 | final val EmptyBytes = Array[Byte]()
53 |
54 | private final val FamilyLoop = new Loop()
55 | private final val QualifierLoop = new Loop()
56 | private final val TimestampLoop = new Loop()
57 |
58 | /** Empty KeyValue (empty row, empty family, empty qualifier, latest timestamp, empty value. */
59 | private final val EmptyKeyValue =
60 | new KeyValue(EmptyBytes, EmptyBytes, EmptyBytes, HConstants.LATEST_TIMESTAMP, EmptyBytes)
61 |
62 | /**
63 | * Builds a Result instance from an actual row content and a get/scan request.
64 | *
65 | * @param table is the fake HTable.
66 | * @param rowKey is the row key.
67 | * @param row is the actual row data.
68 | * @param familyMap is the requested columns.
69 | * @param timeRange is the timestamp range.
70 | * @param maxVersions is the maximum number of versions to return.
71 | * @param filter is an optional HBase filter to apply on the KeyValue entries.
72 | * @return a new Result instance with the specified cells (KeyValue entries).
73 | */
74 | def makeResult(
75 | table: FakeHTable,
76 | rowKey: Bytes,
77 | row: RowFamilies,
78 | familyMap: NavigableMap[Bytes, NavigableSet[Bytes]],
79 | timeRange: TimeRange,
80 | maxVersions: Int,
81 | filter: Filter = PassThroughFilter
82 | ): Result = {
83 |
84 | val cells: JList[Cell] = new JArrayList[Cell]
85 |
86 | /** Current time, in milliseconds, to enforce TTL. */
87 | val nowMS = System.currentTimeMillis
88 |
89 | /**
90 | * Iteration start key.
91 | * Might be updated by filters (see SEEK_NEXT_USING_HINT)
92 | */
93 | var start: Cell = EmptyKeyValue
94 |
95 | /** Ordered set of families to iterate through. */
96 | val families: NavigableSet[Bytes] =
97 | if (!familyMap.isEmpty) familyMap.navigableKeySet else row.navigableKeySet
98 |
99 | /** Iterator over families. */
100 | var family: Bytes = families.ceiling(CellUtil.cloneFamily(start))
101 |
102 | FamilyLoop {
103 | if (family == null) FamilyLoop.break
104 |
105 | /** Map: qualifier -> time series. */
106 | val rowQMap = row.get(family)
107 | if (rowQMap == null) {
108 | family = families.higher(family)
109 | FamilyLoop.continue
110 | }
111 |
112 | // Apply table parameter (TTL, max/min versions):
113 | {
114 | val familyDesc: HColumnDescriptor = table.getFamilyDesc(family)
115 |
116 | val maxVersions = familyDesc.getMaxVersions
117 | val minVersions = familyDesc.getMinVersions
118 | val minTimestamp = nowMS - (familyDesc.getTimeToLive * 1000L)
119 |
120 | for ((qualifier, timeSeries) <- rowQMap.asScalaIterator) {
121 | while (timeSeries.size > maxVersions) {
122 | timeSeries.remove(timeSeries.lastKey)
123 | }
124 | if (familyDesc.getTimeToLive != HConstants.FOREVER) {
125 | while ((timeSeries.size > minVersions)
126 | && (timeSeries.lastKey < minTimestamp)) {
127 | timeSeries.remove(timeSeries.lastKey)
128 | }
129 | }
130 | }
131 | }
132 |
133 | /** Ordered set of qualifiers to iterate through. */
134 | val qualifiers: NavigableSet[Bytes] = {
135 | val reqQSet = familyMap.get(family)
136 | (if ((reqQSet == null) || reqQSet.isEmpty) rowQMap.navigableKeySet else reqQSet)
137 | }
138 |
139 | /** Iterator over the qualifiers. */
140 | var qualifier: Bytes = qualifiers.ceiling(CellUtil.cloneQualifier(start))
141 | QualifierLoop {
142 | if (qualifier == null) QualifierLoop.break
143 |
144 | /** NavigableMap: timestamp -> cell content (Bytes). */
145 | val series: ColumnSeries = rowQMap.get(qualifier)
146 | if (series == null) {
147 | qualifier = qualifiers.higher(qualifier)
148 | QualifierLoop.continue
149 | }
150 |
151 | /** Map: timestamp -> cell value */
152 | val versionMap = series.subMap(timeRange.getMax, false, timeRange.getMin, true)
153 |
154 | /** Ordered set of timestamps to iterate through. */
155 | val timestamps: NavigableSet[JLong] = versionMap.navigableKeySet
156 |
157 | /** Counter to enforce the max-versions per qualifier (post-filtering). */
158 | var nversions = 0
159 |
160 | /** Iterator over the timestamps (decreasing order, ie. most recent first). */
161 | var timestamp: JLong = timestamps.ceiling(start.getTimestamp.asInstanceOf[JLong])
162 | TimestampLoop {
163 | if (timestamp == null) TimestampLoop.break
164 |
165 | val value: Bytes = versionMap.get(timestamp)
166 | val kv: KeyValue = new KeyValue(rowKey, family, qualifier, timestamp, value)
167 |
168 | // Apply filter:
169 | filter.filterKeyValue(kv) match {
170 | case Filter.ReturnCode.INCLUDE => {
171 | cells.add(filter.transformCell(kv))
172 |
173 | // Max-version per qualifier happens after filtering:
174 | // Note that this doesn't take into account the transformed KeyValue.
175 | // It is not clear from the documentation whether filter.transform()
176 | // may alter the qualifier of the KeyValue.
177 | // Also, this doesn't take into account the final modifications resulting from
178 | // filter.filterRow(kvs).
179 | nversions += 1
180 | if (nversions >= maxVersions) {
181 | TimestampLoop.break
182 | }
183 | }
184 | case Filter.ReturnCode.INCLUDE_AND_NEXT_COL => {
185 | cells.add(filter.transformCell(kv))
186 | // No need to check the max-version per qualifier,
187 | // since this jumps to the next column directly.
188 | TimestampLoop.break
189 | }
190 | case Filter.ReturnCode.SKIP => // Skip this key/value pair.
191 | case Filter.ReturnCode.NEXT_COL => TimestampLoop.break
192 | case Filter.ReturnCode.NEXT_ROW => {
193 | // Semantically, NEXT_ROW apparently means NEXT_FAMILY
194 | QualifierLoop.break
195 | }
196 | case Filter.ReturnCode.SEEK_NEXT_USING_HINT => {
197 | Option(filter.getNextCellHint(kv)) match {
198 | case None => // No hint
199 | case Some(hint) => {
200 | require(KeyValueComparator.compare(kv, hint) < 0,
201 | "Filter hint cannot go backward from %s to %s".format(kv, hint))
202 | if (!Arrays.equals(rowKey, hint.getRow)) {
203 | // hint references another row.
204 | //
205 | // For now, this just moves to the next row and the remaining of the
206 | // hint (family/qualifier) is ignored.
207 | // Not sure it makes sense to jump at a specific family/qualifier in
208 | // a different row from a column filter,
209 | // but the documentation does not prevent it.
210 | FamilyLoop.break
211 | }
212 | start = hint
213 | if (!Arrays.equals(family, hint.getFamily)) {
214 | // Restart the family iterator:
215 | family = families.ceiling(hint.getFamily)
216 | FamilyLoop.continue
217 | } else if (!Arrays.equals(qualifier, hint.getQualifier)) {
218 | qualifier = qualifiers.ceiling(hint.getQualifier)
219 | QualifierLoop.continue
220 | } else {
221 | // hint jumps to an older timestamp for the current family/qualifier:
222 | timestamp = timestamps.higher(hint.getTimestamp.asInstanceOf[JLong])
223 | TimestampLoop.continue
224 | }
225 | }
226 | }
227 | }
228 | } // apply filter
229 |
230 | timestamp = timestamps.higher(timestamp).asInstanceOf[JLong]
231 | } // TimestampLoop
232 |
233 | qualifier = qualifiers.higher(qualifier)
234 | } // QualifierLoop
235 |
236 | family = families.higher(family)
237 | } // FamilyLoop
238 |
239 | if (filter.hasFilterRow()) {
240 | filter.filterRowCells(cells)
241 | }
242 | return Result.create(cells)
243 | }
244 |
245 | }
--------------------------------------------------------------------------------
/LICENSE.txt:
--------------------------------------------------------------------------------
1 |
2 | Apache License
3 | Version 2.0, January 2004
4 | http://www.apache.org/licenses/
5 |
6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
7 |
8 | 1. Definitions.
9 |
10 | "License" shall mean the terms and conditions for use, reproduction,
11 | and distribution as defined by Sections 1 through 9 of this document.
12 |
13 | "Licensor" shall mean the copyright owner or entity authorized by
14 | the copyright owner that is granting the License.
15 |
16 | "Legal Entity" shall mean the union of the acting entity and all
17 | other entities that control, are controlled by, or are under common
18 | control with that entity. For the purposes of this definition,
19 | "control" means (i) the power, direct or indirect, to cause the
20 | direction or management of such entity, whether by contract or
21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
22 | outstanding shares, or (iii) beneficial ownership of such entity.
23 |
24 | "You" (or "Your") shall mean an individual or Legal Entity
25 | exercising permissions granted by this License.
26 |
27 | "Source" form shall mean the preferred form for making modifications,
28 | including but not limited to software source code, documentation
29 | source, and configuration files.
30 |
31 | "Object" form shall mean any form resulting from mechanical
32 | transformation or translation of a Source form, including but
33 | not limited to compiled object code, generated documentation,
34 | and conversions to other media types.
35 |
36 | "Work" shall mean the work of authorship, whether in Source or
37 | Object form, made available under the License, as indicated by a
38 | copyright notice that is included in or attached to the work
39 | (an example is provided in the Appendix below).
40 |
41 | "Derivative Works" shall mean any work, whether in Source or Object
42 | form, that is based on (or derived from) the Work and for which the
43 | editorial revisions, annotations, elaborations, or other modifications
44 | represent, as a whole, an original work of authorship. For the purposes
45 | of this License, Derivative Works shall not include works that remain
46 | separable from, or merely link (or bind by name) to the interfaces of,
47 | the Work and Derivative Works thereof.
48 |
49 | "Contribution" shall mean any work of authorship, including
50 | the original version of the Work and any modifications or additions
51 | to that Work or Derivative Works thereof, that is intentionally
52 | submitted to Licensor for inclusion in the Work by the copyright owner
53 | or by an individual or Legal Entity authorized to submit on behalf of
54 | the copyright owner. For the purposes of this definition, "submitted"
55 | means any form of electronic, verbal, or written communication sent
56 | to the Licensor or its representatives, including but not limited to
57 | communication on electronic mailing lists, source code control systems,
58 | and issue tracking systems that are managed by, or on behalf of, the
59 | Licensor for the purpose of discussing and improving the Work, but
60 | excluding communication that is conspicuously marked or otherwise
61 | designated in writing by the copyright owner as "Not a Contribution."
62 |
63 | "Contributor" shall mean Licensor and any individual or Legal Entity
64 | on behalf of whom a Contribution has been received by Licensor and
65 | subsequently incorporated within the Work.
66 |
67 | 2. Grant of Copyright License. Subject to the terms and conditions of
68 | this License, each Contributor hereby grants to You a perpetual,
69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
70 | copyright license to reproduce, prepare Derivative Works of,
71 | publicly display, publicly perform, sublicense, and distribute the
72 | Work and such Derivative Works in Source or Object form.
73 |
74 | 3. Grant of Patent License. Subject to the terms and conditions of
75 | this License, each Contributor hereby grants to You a perpetual,
76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
77 | (except as stated in this section) patent license to make, have made,
78 | use, offer to sell, sell, import, and otherwise transfer the Work,
79 | where such license applies only to those patent claims licensable
80 | by such Contributor that are necessarily infringed by their
81 | Contribution(s) alone or by combination of their Contribution(s)
82 | with the Work to which such Contribution(s) was submitted. If You
83 | institute patent litigation against any entity (including a
84 | cross-claim or counterclaim in a lawsuit) alleging that the Work
85 | or a Contribution incorporated within the Work constitutes direct
86 | or contributory patent infringement, then any patent licenses
87 | granted to You under this License for that Work shall terminate
88 | as of the date such litigation is filed.
89 |
90 | 4. Redistribution. You may reproduce and distribute copies of the
91 | Work or Derivative Works thereof in any medium, with or without
92 | modifications, and in Source or Object form, provided that You
93 | meet the following conditions:
94 |
95 | (a) You must give any other recipients of the Work or
96 | Derivative Works a copy of this License; and
97 |
98 | (b) You must cause any modified files to carry prominent notices
99 | stating that You changed the files; and
100 |
101 | (c) You must retain, in the Source form of any Derivative Works
102 | that You distribute, all copyright, patent, trademark, and
103 | attribution notices from the Source form of the Work,
104 | excluding those notices that do not pertain to any part of
105 | the Derivative Works; and
106 |
107 | (d) If the Work includes a "NOTICE" text file as part of its
108 | distribution, then any Derivative Works that You distribute must
109 | include a readable copy of the attribution notices contained
110 | within such NOTICE file, excluding those notices that do not
111 | pertain to any part of the Derivative Works, in at least one
112 | of the following places: within a NOTICE text file distributed
113 | as part of the Derivative Works; within the Source form or
114 | documentation, if provided along with the Derivative Works; or,
115 | within a display generated by the Derivative Works, if and
116 | wherever such third-party notices normally appear. The contents
117 | of the NOTICE file are for informational purposes only and
118 | do not modify the License. You may add Your own attribution
119 | notices within Derivative Works that You distribute, alongside
120 | or as an addendum to the NOTICE text from the Work, provided
121 | that such additional attribution notices cannot be construed
122 | as modifying the License.
123 |
124 | You may add Your own copyright statement to Your modifications and
125 | may provide additional or different license terms and conditions
126 | for use, reproduction, or distribution of Your modifications, or
127 | for any such Derivative Works as a whole, provided Your use,
128 | reproduction, and distribution of the Work otherwise complies with
129 | the conditions stated in this License.
130 |
131 | 5. Submission of Contributions. Unless You explicitly state otherwise,
132 | any Contribution intentionally submitted for inclusion in the Work
133 | by You to the Licensor shall be under the terms and conditions of
134 | this License, without any additional terms or conditions.
135 | Notwithstanding the above, nothing herein shall supersede or modify
136 | the terms of any separate license agreement you may have executed
137 | with Licensor regarding such Contributions.
138 |
139 | 6. Trademarks. This License does not grant permission to use the trade
140 | names, trademarks, service marks, or product names of the Licensor,
141 | except as required for reasonable and customary use in describing the
142 | origin of the Work and reproducing the content of the NOTICE file.
143 |
144 | 7. Disclaimer of Warranty. Unless required by applicable law or
145 | agreed to in writing, Licensor provides the Work (and each
146 | Contributor provides its Contributions) on an "AS IS" BASIS,
147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148 | implied, including, without limitation, any warranties or conditions
149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150 | PARTICULAR PURPOSE. You are solely responsible for determining the
151 | appropriateness of using or redistributing the Work and assume any
152 | risks associated with Your exercise of permissions under this License.
153 |
154 | 8. Limitation of Liability. In no event and under no legal theory,
155 | whether in tort (including negligence), contract, or otherwise,
156 | unless required by applicable law (such as deliberate and grossly
157 | negligent acts) or agreed to in writing, shall any Contributor be
158 | liable to You for damages, including any direct, indirect, special,
159 | incidental, or consequential damages of any character arising as a
160 | result of this License or out of the use or inability to use the
161 | Work (including but not limited to damages for loss of goodwill,
162 | work stoppage, computer failure or malfunction, or any and all
163 | other commercial damages or losses), even if such Contributor
164 | has been advised of the possibility of such damages.
165 |
166 | 9. Accepting Warranty or Additional Liability. While redistributing
167 | the Work or Derivative Works thereof, You may choose to offer,
168 | and charge a fee for, acceptance of support, warranty, indemnity,
169 | or other liability obligations and/or rights consistent with this
170 | License. However, in accepting such obligations, You may act only
171 | on Your own behalf and on Your sole responsibility, not on behalf
172 | of any other Contributor, and only if You agree to indemnify,
173 | defend, and hold each Contributor harmless for any liability
174 | incurred by, or claims asserted against, such Contributor by reason
175 | of your accepting any such warranty or additional liability.
176 |
177 | END OF TERMS AND CONDITIONS
178 |
179 | APPENDIX: How to apply the Apache License to your work.
180 |
181 | To apply the Apache License to your work, attach the following
182 | boilerplate notice, with the fields enclosed by brackets "[]"
183 | replaced with your own identifying information. (Don't include
184 | the brackets!) The text should be enclosed in the appropriate
185 | comment syntax for the file format. We also recommend that a
186 | file or class name and description of purpose be included on the
187 | same "printed page" as the copyright notice for easier
188 | identification within third-party archives.
189 |
190 | Copyright [yyyy] [name of copyright owner]
191 |
192 | Licensed under the Apache License, Version 2.0 (the "License");
193 | you may not use this file except in compliance with the License.
194 | You may obtain a copy of the License at
195 |
196 | http://www.apache.org/licenses/LICENSE-2.0
197 |
198 | Unless required by applicable law or agreed to in writing, software
199 | distributed under the License is distributed on an "AS IS" BASIS,
200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
201 | See the License for the specific language governing permissions and
202 | limitations under the License.
203 |
--------------------------------------------------------------------------------
/src/test/scala/org/kiji/testing/fakehtable/TestFakeHTable.scala:
--------------------------------------------------------------------------------
1 | /**
2 | * (c) Copyright 2012 WibiData, Inc.
3 | *
4 | * See the NOTICE file distributed with this work for additional
5 | * information regarding copyright ownership.
6 | *
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | */
19 |
20 | package org.kiji.testing.fakehtable
21 |
22 | import java.util.Arrays
23 | import scala.collection.JavaConverters.asScalaIteratorConverter
24 | import scala.collection.JavaConverters.asScalaSetConverter
25 | import scala.collection.JavaConverters.mapAsScalaMapConverter
26 | import org.apache.hadoop.hbase.HBaseConfiguration
27 | import org.apache.hadoop.hbase.HColumnDescriptor
28 | import org.apache.hadoop.hbase.HConstants
29 | import org.apache.hadoop.hbase.HTableDescriptor
30 | import org.apache.hadoop.hbase.client.Append
31 | import org.apache.hadoop.hbase.client.Delete
32 | import org.apache.hadoop.hbase.client.Get
33 | import org.apache.hadoop.hbase.client.HTableInterface
34 | import org.apache.hadoop.hbase.client.Put
35 | import org.apache.hadoop.hbase.client.Scan
36 | import org.apache.hadoop.hbase.filter.ColumnPrefixFilter
37 | import org.apache.hadoop.hbase.filter.ColumnRangeFilter
38 | import org.apache.hadoop.hbase.filter.KeyOnlyFilter
39 | import org.apache.hadoop.hbase.util.Bytes
40 | import org.junit.Assert
41 | import org.junit.Test
42 | import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter
43 | import org.slf4j.LoggerFactory
44 |
45 | class TestFakeHTable {
46 | private final val Log = LoggerFactory.getLogger(classOf[TestFakeHTable])
47 |
48 | type Bytes = Array[Byte]
49 |
50 | /**
51 | * Implicitly converts string to UTF-8 bytes.
52 | *
53 | * @param string String to convert to bytes.
54 | * @return the string as UTF-8 encoded bytes.
55 | */
56 | implicit def stringToBytes(string: String): Bytes = {
57 | return Bytes.toBytes(string)
58 | }
59 |
60 | /**
61 | * Decodes UTF-8 encoded bytes to a string.
62 | *
63 | * @param bytes UTF-8 encoded bytes.
64 | * @return the decoded string.
65 | */
66 | def bytesToString(bytes: Bytes): String = {
67 | return Bytes.toString(bytes)
68 | }
69 |
70 | /**
71 | * Implicit string wrapper to add convenience encoding methods
72 | *
73 | *
74 | * Allows to write "the string"b to represent the UTF-8 bytes for "the string".
75 | *
76 | *
77 | * @param str String to wrap.
78 | */
79 | class StringAsBytes(str: String) {
80 | /** Encodes the string to UTF-8 bytes. */
81 | def bytes(): Bytes = stringToBytes(str)
82 |
83 | /** Shortcut for bytes(). */
84 | def b = bytes()
85 | }
86 |
87 | implicit def implicitToStringAsBytes(str: String): StringAsBytes = {
88 | return new StringAsBytes(str)
89 | }
90 |
91 | // -----------------------------------------------------------------------------------------------
92 |
93 | /** Table descriptor for a test table with an HBase family named "family". */
94 | def defaultTableDesc(): HTableDescriptor = {
95 | val desc = new HTableDescriptor("table")
96 | desc.addFamily(new HColumnDescriptor("family")
97 | .setMaxVersions(HConstants.ALL_VERSIONS)
98 | .setMinVersions(0)
99 | .setTimeToLive(HConstants.FOREVER)
100 | .setInMemory(false)
101 | )
102 | desc
103 | }
104 |
105 | /** Table descriptor for a test table with N HBase families named "family". */
106 | def makeTableDesc(nfamilies: Int): HTableDescriptor = {
107 | val desc = new HTableDescriptor("table")
108 | for (ifamily <- 0 until nfamilies) {
109 | desc.addFamily(new HColumnDescriptor("family%s".format(ifamily))
110 | .setMaxVersions(HConstants.ALL_VERSIONS)
111 | .setMinVersions(0)
112 | .setTimeToLive(HConstants.FOREVER)
113 | .setInMemory(false)
114 | )
115 | }
116 | desc
117 | }
118 |
119 | // -----------------------------------------------------------------------------------------------
120 |
121 | /** get() on an unknown row. */
122 | @Test
123 | def testGetUnknownRow(): Unit = {
124 | val table = new FakeHTable(name = "table", desc = defaultTableDesc)
125 | Assert.assertEquals(true, table.get(new Get("key")).isEmpty)
126 | }
127 |
128 | /** scan() on an empty table. */
129 | @Test
130 | def testScanEmptyTable(): Unit = {
131 | val table = new FakeHTable(name = "table", desc = defaultTableDesc)
132 | Assert.assertEquals(null, table.getScanner("family").next)
133 | }
134 |
135 | /** delete() on an unknown row. */
136 | @Test
137 | def testDeleteUnknownRow(): Unit = {
138 | val table = new FakeHTable(name = "table", desc = defaultTableDesc)
139 | table.delete(new Delete("key"))
140 | }
141 |
142 | /** Write a few cells and read them back. */
143 | @Test
144 | def testPutThenGet(): Unit = {
145 | val table = new FakeHTable(name = "table", desc = defaultTableDesc)
146 | table.put(new Put("key")
147 | .add("family", "qualifier", 12345L, "value"))
148 |
149 | val result = table.get(new Get("key"))
150 | Assert.assertEquals(false, result.isEmpty)
151 | Assert.assertEquals("key", Bytes.toString(result.getRow))
152 | Assert.assertEquals("value", Bytes.toString(result.value))
153 |
154 | Assert.assertEquals(1, result.getMap.size)
155 | Assert.assertEquals(1, result.getMap.get("family"b).size)
156 | Assert.assertEquals(1, result.getMap.get("family"b).get("qualifier"b).size)
157 | Assert.assertEquals(
158 | "value",
159 | Bytes.toString(result.getMap.get("family"b).get("qualifier"b).get(12345L)))
160 | }
161 |
162 | /** Write a few cells and read them back as a family scan. */
163 | @Test
164 | def testPutThenScan(): Unit = {
165 | val table = new FakeHTable(name = "table", desc = defaultTableDesc)
166 | table.put(new Put("key")
167 | .add("family", "qualifier", 12345L, "value"))
168 |
169 | val scanner = table.getScanner("family")
170 | val it = scanner.iterator
171 | Assert.assertEquals(true, it.hasNext)
172 | val result = it.next
173 | Assert.assertEquals(false, result.isEmpty)
174 | Assert.assertEquals("key", Bytes.toString(result.getRow))
175 | Assert.assertEquals("value", Bytes.toString(result.value))
176 |
177 | Assert.assertEquals(1, result.getMap.size)
178 | Assert.assertEquals(1, result.getMap.get("family"b).size)
179 | Assert.assertEquals(1, result.getMap.get("family"b).get("qualifier"b).size)
180 | Assert.assertEquals(
181 | "value",
182 | Bytes.toString(result.getMap.get("family"b).get("qualifier"b).get(12345L)))
183 |
184 | Assert.assertEquals(false, it.hasNext)
185 | }
186 |
187 | /** Create a row and delete it. */
188 | @Test
189 | def testCreateAndDeleteRow(): Unit = {
190 | val table = new FakeHTable(name = "table", desc = defaultTableDesc)
191 | table.put(new Put("key")
192 | .add("family", "qualifier", 12345L, "value"))
193 |
194 | table.delete(new Delete("key"))
195 | Assert.assertEquals(true, table.get(new Get("key")).isEmpty)
196 | }
197 |
198 | /** Increment a column. */
199 | @Test
200 | def testIncrementColumn(): Unit = {
201 | val table = new FakeHTable(name = "table", desc = defaultTableDesc)
202 | Assert.assertEquals(1, table.incrementColumnValue(
203 | row = "row",
204 | family = "family",
205 | qualifier = "qualifier",
206 | amount = 1))
207 | Assert.assertEquals(2, table.incrementColumnValue(
208 | row = "row",
209 | family = "family",
210 | qualifier = "qualifier",
211 | amount = 1))
212 | Assert.assertEquals(3, table.incrementColumnValue(
213 | row = "row",
214 | family = "family",
215 | qualifier = "qualifier",
216 | amount = 1))
217 | }
218 |
219 | /** Delete a specific cell. */
220 | @Test
221 | def testDeleteSpecificCell(): Unit = {
222 | val table = new FakeHTable(name = "table", desc = defaultTableDesc)
223 | table.put(new Put("key")
224 | .add("family", "qualifier", 1L, "value1"))
225 | table.put(new Put("key")
226 | .add("family", "qualifier", 2L, "value2"))
227 | table.put(new Put("key")
228 | .add("family", "qualifier", 3L, "value3"))
229 | table.delete(new Delete("key")
230 | .deleteColumn("family", "qualifier", 2L))
231 | val scanner = table.getScanner(new Scan("key")
232 | .setMaxVersions(Int.MaxValue)
233 | .addColumn("family", "qualifier"))
234 | val row = scanner.next()
235 | Assert.assertEquals(null, scanner.next())
236 | val cells = row.getColumn("family", "qualifier")
237 | Assert.assertEquals(2, cells.size())
238 | Assert.assertEquals(3L, cells.get(0).getTimestamp)
239 | Assert.assertEquals(1L, cells.get(1).getTimestamp)
240 | }
241 |
242 | /** Delete the most recent cell in a column. */
243 | @Test
244 | def testDeleteMostRecentCell(): Unit = {
245 | val table = new FakeHTable(name = "table", desc = defaultTableDesc)
246 | table.put(new Put("key")
247 | .add("family", "qualifier", 1L, "value1"))
248 | table.put(new Put("key")
249 | .add("family", "qualifier", 2L, "value2"))
250 | table.put(new Put("key")
251 | .add("family", "qualifier", 3L, "value3"))
252 |
253 | if (Log.isDebugEnabled)
254 | table.dump()
255 |
256 | table.delete(new Delete("key")
257 | .deleteColumn("family", "qualifier"))
258 |
259 | if (Log.isDebugEnabled)
260 | table.dump()
261 |
262 | val row = {
263 | if (false) {
264 | val scanner = table.getScanner(new Scan("key")
265 | .setMaxVersions(Int.MaxValue)
266 | .addColumn("family", "qualifier"))
267 | val row = scanner.next()
268 | Assert.assertEquals(null, scanner.next())
269 | row
270 | } else {
271 | table.get(new Get("key")
272 | .setMaxVersions(Int.MaxValue)
273 | .addColumn("family", "qualifier"))
274 | }
275 | }
276 |
277 | val cells = row.getColumnCells("family", "qualifier")
278 | Assert.assertEquals(2, cells.size())
279 | Assert.assertEquals(2L, cells.get(0).getTimestamp)
280 | Assert.assertEquals(1L, cells.get(1).getTimestamp)
281 | }
282 |
283 | /** Delete older versions of a qualifier. */
284 | @Test
285 | def testDeleteOlderCellVersions(): Unit = {
286 | val table = new FakeHTable(name = "table", desc = defaultTableDesc)
287 | table.put(new Put("key")
288 | .add("family", "qualifier", 1L, "value1"))
289 | table.put(new Put("key")
290 | .add("family", "qualifier", 2L, "value2"))
291 | table.put(new Put("key")
292 | .add("family", "qualifier", 3L, "value3"))
293 | table.delete(new Delete("key")
294 | .deleteColumns("family", "qualifier", 2L))
295 | val scanner = table.getScanner(new Scan("key")
296 | .setMaxVersions(Int.MaxValue)
297 | .addColumn("family", "qualifier"))
298 | val row = scanner.next()
299 | Assert.assertEquals(null, scanner.next())
300 | val cells = row.getColumn("family", "qualifier")
301 | Assert.assertEquals(1, cells.size())
302 | Assert.assertEquals(3L, cells.get(0).getTimestamp)
303 | }
304 |
305 | /** Delete all versions of a specific qualifier. */
306 | @Test
307 | def testDeleteSpecificQualifierAllVersions(): Unit = {
308 | val table = new FakeHTable(name = "table", desc = defaultTableDesc)
309 | table.put(new Put("key")
310 | .add("family", "qualifier", 1L, "value1"))
311 | table.put(new Put("key")
312 | .add("family", "qualifier", 2L, "value2"))
313 | table.put(new Put("key")
314 | .add("family", "qualifier", 3L, "value3"))
315 | table.delete(new Delete("key")
316 | .deleteColumns("family", "qualifier"))
317 | val scanner = table.getScanner(new Scan("key")
318 | .setMaxVersions(Int.MaxValue)
319 | .addColumn("family", "qualifier"))
320 | Assert.assertEquals(null, scanner.next())
321 | }
322 |
323 | /** Test that families are cleaned up properly when the last qualifier disappears. */
324 | @Test
325 | def testFamilyCleanupAfterDelete(): Unit = {
326 | val table = new FakeHTable(name = "table", desc = makeTableDesc(nfamilies = 4))
327 |
328 | // Populate one row with 4 families, each with 4 qualifiers, each with 4 versions:
329 | val count = 4
330 | val rowKey = "key1"
331 | populateTable(table, count = count)
332 |
333 | // Delete all versions of family1:qualifier1 one by one, and check:
334 | for (timestamp <- 0 until count) {
335 | table.delete(new Delete(rowKey)
336 | .deleteColumn("family1", "qualifier1", timestamp))
337 | }
338 | {
339 | val scanner = table.getScanner(new Scan(rowKey, nextRow(rowKey))
340 | .setMaxVersions(Int.MaxValue)
341 | .addColumn("family1", "qualifier1"))
342 | Assert.assertEquals(null, scanner.next())
343 | }
344 |
345 | // Delete all qualifiers in family2 and check:
346 | for (cId <- 0 until count) {
347 | table.delete(new Delete(rowKey)
348 | .deleteColumns("family2", "qualifier%d".format(cId)))
349 | }
350 | {
351 | val scanner = table.getScanner(new Scan(rowKey, nextRow(rowKey))
352 | .setMaxVersions(Int.MaxValue)
353 | .addFamily("family2"))
354 | Assert.assertEquals(null, scanner.next())
355 | }
356 | }
357 |
358 | /** Test ResultScanner.hasNext() on a empty table. */
359 | @Test
360 | def testResultScannerHasNextOnEmptyTable(): Unit = {
361 | val table = new FakeHTable(name = "table", desc = defaultTableDesc)
362 | val scanner = table.getScanner(new Scan())
363 | val iterator = scanner.iterator()
364 | Assert.assertEquals(false, iterator.hasNext())
365 | Assert.assertEquals(null, iterator.next())
366 | }
367 |
368 | /** Test ResultScanner.hasNext() with stop row-key on an empty table. */
369 | @Test
370 | def testResultScannerWithStopRowKeyOnEmptyTable(): Unit = {
371 | val table = new FakeHTable(name = "table", desc = defaultTableDesc)
372 | val scanner = table.getScanner(new Scan().setStopRow("stop"))
373 | val iterator = scanner.iterator()
374 | Assert.assertEquals(false, iterator.hasNext())
375 | Assert.assertEquals(null, iterator.next())
376 | }
377 |
378 | /** Test ResultScanner.hasNext() while scanning a full table. */
379 | @Test
380 | def testResultScannerHasNext(): Unit = {
381 | val table = new FakeHTable(name = "table", desc = defaultTableDesc)
382 | table.put(new Put("key")
383 | .add("family", "qualifier", 1L, "value1"))
384 |
385 | val scanner = table.getScanner(new Scan())
386 | val iterator = scanner.iterator()
387 | Assert.assertEquals(true, iterator.hasNext())
388 | assert(iterator.next() != null)
389 | Assert.assertEquals(false, iterator.hasNext())
390 | Assert.assertEquals(null, iterator.next())
391 | }
392 |
393 | /** Test ResultScanner.hasNext() while scanning a specific column. */
394 | @Test
395 | def testResultScannerHasNextWithQualifier(): Unit = {
396 | val table = new FakeHTable(name = "table", desc = defaultTableDesc)
397 | table.put(new Put("key1")
398 | .add("family", "qualifier1", 1L, "value1"))
399 | table.put(new Put("key2")
400 | .add("family", "qualifier2", 1L, "value1"))
401 |
402 | val scanner = table.getScanner(new Scan()
403 | .addColumn("family", "qualifier1"))
404 | val iterator = scanner.iterator()
405 | Assert.assertEquals(true, iterator.hasNext())
406 | assert(iterator.next() != null)
407 | Assert.assertEquals(false, iterator.hasNext())
408 | Assert.assertEquals(null, iterator.next())
409 | }
410 |
411 | /** Test a get request with a filter. */
412 | @Test
413 | def testGetWithFilter(): Unit = {
414 | val table = new FakeHTable(
415 | name = "table",
416 | conf = HBaseConfiguration.create(),
417 | desc = makeTableDesc(nfamilies = 2)
418 | )
419 |
420 | val count = 2
421 | populateTable(table, count=count)
422 |
423 | // if (Log.isDebugEnabled)
424 | table.dump()
425 |
426 | val get = new Get("key1")
427 | .setMaxVersions()
428 | .setFilter(new KeyOnlyFilter)
429 | val result = table.get(get)
430 | Assert.assertEquals(count, result.getMap.size)
431 | for ((family, qmap) <- result.getMap.asScala) {
432 | Assert.assertEquals(count, qmap.size)
433 | for ((qualifier, tseries) <- qmap.asScala) {
434 | Assert.assertEquals(count, tseries.size)
435 | for ((timestamp, value) <- tseries.asScala) {
436 | Assert.assertEquals(0, value.size)
437 | }
438 | }
439 | }
440 | }
441 |
442 | /**
443 | * Test a get request with the FirstKeyOnly filter.
444 | *
445 | * This validates some behaviors around ReturnCode.NEXT_ROW.
446 | */
447 | @Test
448 | def testGetWithFirstKeyOnlyFilter(): Unit = {
449 | val table = new FakeHTable(
450 | name = "table",
451 | conf = HBaseConfiguration.create(),
452 | desc = makeTableDesc(nfamilies = 2)
453 | )
454 |
455 | val count = 2
456 | populateTable(table, count=count)
457 |
458 | val get = new Get("key1")
459 | .setMaxVersions()
460 | .setFilter(new FirstKeyOnlyFilter)
461 | val result = table.get(get)
462 |
463 | Assert.assertEquals(1, result.getMap.size)
464 | Assert.assertTrue(result.containsColumn("family0", "qualifier0"))
465 | }
466 |
467 | /** Test a get request with max versions. */
468 | @Test
469 | def testGetWithMaxVersions(): Unit = {
470 | val table = new FakeHTable(
471 | name = "table",
472 | conf = HBaseConfiguration.create(),
473 | desc = makeTableDesc(nfamilies = 4)
474 | )
475 |
476 | val count = 4
477 | populateTable(table, count=count)
478 |
479 | // We generated 4 versions, but request only 2:
480 | val maxVersions = 2
481 | val get = new Get("key1")
482 | .setMaxVersions(maxVersions)
483 | // .setFilter(new KeyOnlyFilter)
484 | val result = table.get(get)
485 | Assert.assertEquals(count, result.getMap.size)
486 | for ((family, qmap) <- result.getMap.asScala) {
487 | Assert.assertEquals(count, qmap.size)
488 | for ((qualifier, tseries) <- qmap.asScala) {
489 | Assert.assertEquals(maxVersions, tseries.size)
490 | assert(tseries.containsKey(2L))
491 | assert(tseries.containsKey(3L))
492 | }
493 | }
494 | }
495 |
496 | /** Test a scan with an explicit start row. */
497 | @Test
498 | def testScanWithStartRow(): Unit = {
499 | val table = new FakeHTable(
500 | name = "table",
501 | conf = HBaseConfiguration.create(),
502 | desc = makeTableDesc(nfamilies = 3)
503 | )
504 |
505 | val count = 3
506 | populateTable(table, count=count)
507 |
508 | {
509 | val scanner = table.getScanner(new Scan().setStartRow("key1"))
510 | val rows = scanner.iterator().asScala.toList
511 | Assert.assertEquals(2, rows.size)
512 | Assert.assertEquals("key1", Bytes.toString(rows(0).getRow))
513 | Assert.assertEquals("key2", Bytes.toString(rows(1).getRow))
514 | }
515 | {
516 | val scanner = table.getScanner(new Scan().setStartRow("key1a"))
517 | val rows = scanner.iterator().asScala.toList
518 | Assert.assertEquals(1, rows.size)
519 | Assert.assertEquals("key2", Bytes.toString(rows(0).getRow))
520 | }
521 | }
522 |
523 | /** Test scanning with a prefix filter. */
524 | @Test
525 | def testScanWithFilter(): Unit = {
526 | val table = new FakeHTable(
527 | name = "table",
528 | conf = HBaseConfiguration.create(),
529 | desc = defaultTableDesc
530 | )
531 |
532 | val rowKey = "key"
533 | val family = "family"
534 | for (qualifier <- List("non-prefixed", "prefix:a")) {
535 | table.put(new Put(rowKey).add(family, qualifier, qualifier))
536 | }
537 |
538 | {
539 | val get = new Get(rowKey).setFilter(new ColumnPrefixFilter("prefix"))
540 | val result = table.get(get)
541 | val values = result.raw().toList.map(kv => Bytes.toString(kv.getValue))
542 | Assert.assertEquals(List("prefix:a"), values)
543 | }
544 |
545 | {
546 | val scan = new Scan(rowKey).setFilter(new ColumnPrefixFilter("prefix"))
547 | val scanner = table.getScanner(scan)
548 | val rows = scanner.iterator().asScala.toList
549 |
550 | val values = rows(0).raw().toList.map(kv => Bytes.toString(kv.getValue))
551 | Assert.assertEquals(List("prefix:a"), values)
552 | }
553 | }
554 |
555 | /**
556 | * Test a get request with ColumnRangefilter.
557 | * Range is inclusive and yields a non-empty result.
558 | */
559 | @Test
560 | def testColumnRangeFilterWithInclusiveBound(): Unit = {
561 | val table = new FakeHTable(
562 | name = "table",
563 | conf = HBaseConfiguration.create(),
564 | desc = makeTableDesc(nfamilies = 5)
565 | )
566 | val count = 5
567 | for (x <- 0 until count) {
568 | val familyDesc = new HColumnDescriptor("family%d".format(x))
569 | .setMaxVersions(HConstants.ALL_VERSIONS)
570 | table.getTableDescriptor.addFamily(familyDesc)
571 | }
572 | populateTable(table, count=count)
573 |
574 | // Only fetch qualifiers >= 'qualifier2' and <= 'qualifier3',
575 | // this should be exactly "qualifier2" and "qualifier3":
576 | val get = new Get("key1")
577 | .setFilter(new ColumnRangeFilter("qualifier2", true, "qualifier3", true))
578 | .setMaxVersions()
579 |
580 | val result = table.get(get)
581 | Assert.assertEquals("key1", bytesToString(result.getRow))
582 | val families = for (x <- 0 until count) yield "family%d".format(x)
583 | Assert.assertEquals(families, result.getMap.keySet.asScala.toList.map {bytesToString(_)})
584 | for (family <- families) {
585 | val qmap = result.getMap.get(family.bytes())
586 | Assert.assertEquals(List("qualifier2", "qualifier3"), qmap.keySet.asScala.toList.map {Bytes.toString(_)})
587 | Assert.assertEquals(5, qmap.firstEntry.getValue.size)
588 | Assert.assertEquals(5, qmap.lastEntry.getValue.size)
589 | }
590 | }
591 |
592 | /**
593 | * Test a get request with a ColumnRangeFilter.
594 | * Range is exclusive and yields a non-empty result.
595 | */
596 | @Test
597 | def testColumnRangeFilterWithExclusiveBoundWithResult(): Unit = {
598 | val table = new FakeHTable(
599 | name = "table",
600 | conf = HBaseConfiguration.create(),
601 | desc = makeTableDesc(nfamilies = 5)
602 | )
603 | val count = 5
604 | for (x <- 0 until count) {
605 | val familyDesc = new HColumnDescriptor("family%d".format(x))
606 | .setMaxVersions(HConstants.ALL_VERSIONS)
607 | table.getTableDescriptor.addFamily(familyDesc)
608 | }
609 | populateTable(table, count=count)
610 |
611 | // Only fetch qualifiers > 'qualifier2' and < 'qualifier4',
612 | // this should be exactly "qualifier3":
613 | val get = new Get("key1")
614 | .setFilter(
615 | new ColumnRangeFilter("qualifier2", false, "qualifier4", false))
616 | .setMaxVersions()
617 |
618 | val result = table.get(get)
619 | Assert.assertEquals("key1", bytesToString(result.getRow))
620 | val families = for (x <- 0 until count) yield "family%d".format(x)
621 | Assert.assertEquals(families, result.getMap.keySet.asScala.toList.map {Bytes.toString(_)})
622 | for (family <- families) {
623 | val qmap = result.getMap.get(family.bytes())
624 | Assert.assertEquals(List("qualifier3"), qmap.keySet.asScala.toList.map {Bytes.toString(_)})
625 | Assert.assertEquals(5, qmap.firstEntry.getValue.size)
626 | }
627 | }
628 |
629 | /**
630 | * Test a get request with a ColumnRangeFilter.
631 | * Range is exclusive and yields an empty result.
632 | */
633 | @Test
634 | def testColumnRangeFilterWithExclusiveBoundAndEmptyResult(): Unit = {
635 | val table = new FakeHTable(
636 | name = "table",
637 | conf = HBaseConfiguration.create(),
638 | desc = makeTableDesc(nfamilies = 5)
639 | )
640 | val count = 5
641 | populateTable(table, count=count)
642 |
643 | // Only fetch qualifiers >= 'qualifier2.5' and < 'qualifier3',
644 | // this should be empty:
645 | val get = new Get("key1")
646 | .setFilter(
647 | new ColumnRangeFilter("qualifier2.5", true, "qualifier3", false))
648 | .setMaxVersions()
649 |
650 | val result = table.get(get)
651 | Assert.assertEquals(true, result.isEmpty)
652 | }
653 |
654 | /** Test that max versions applies correctly while putting many cells. */
655 | @Test
656 | def testMaxVersions(): Unit = {
657 | val desc = new HTableDescriptor("table")
658 | desc.addFamily(new HColumnDescriptor("family")
659 | .setMaxVersions(5)
660 | // min versions defaults to 0, TTL defaults to forever.
661 | )
662 | val table = new FakeHTable(
663 | name = "table",
664 | conf = HBaseConfiguration.create(),
665 | desc = desc
666 | )
667 |
668 | for (index <- 0 until 9) {
669 | val timestamp = index
670 | table.put(new Put("row").add("family", "q", timestamp, "value-%d".format(index)))
671 | }
672 |
673 | val result = table.get(new Get("row").addFamily("family").setMaxVersions())
674 | val kvs = result.getColumn("family", "q")
675 | Assert.assertEquals(5, kvs.size)
676 | }
677 |
678 | /** Test that TTL applies correctly with no min versions (ie. min versions = 0). */
679 | @Test
680 | def testTTLWithoutMinVersion(): Unit = {
681 | val ttl = 3600 // 1h TTL
682 |
683 | val desc = new HTableDescriptor("table")
684 | desc.addFamily(new HColumnDescriptor("family")
685 | .setMaxVersions(HConstants.ALL_VERSIONS) // retain all versions
686 | .setTimeToLive(ttl) // 1h TTL
687 | .setMinVersions(0) // no min versions to retain wrt TTL
688 | )
689 | val table = new FakeHTable(
690 | name = "table",
691 | conf = HBaseConfiguration.create(),
692 | desc = desc
693 | )
694 |
695 | val nowMS = System.currentTimeMillis
696 | val minTimestamp = nowMS - (ttl * 1000)
697 |
698 | // Write cells older than TTL, all these cells should be discarded:
699 | for (index <- 0 until 9) {
700 | val timestamp = minTimestamp - index // absolutely older than TTL allows
701 | table.put(new Put("row").add("family", "q", timestamp, "value-%d".format(index)))
702 | }
703 |
704 | {
705 | val result = table.get(new Get("row").addFamily("family").setMaxVersions())
706 | val kvs = result.getColumn("family", "q")
707 | // Must be empty since all the puts were older than TTL allows and no min versions set:
708 | Assert.assertEquals(0, kvs.size)
709 | }
710 |
711 | // Write cells within TTL range, all these cells should be kept:
712 | for (index <- 0 until 9) {
713 | val timestamp = nowMS + index // within TTL range
714 | table.put(new Put("row").add("family", "q", timestamp, "value-%d".format(index)))
715 | }
716 |
717 | {
718 | val result = table.get(new Get("row").addFamily("family").setMaxVersions())
719 | val kvs = result.getColumn("family", "q")
720 | Assert.assertEquals(9, kvs.size)
721 | }
722 | }
723 |
724 | /** Test that the min versions is respected while max TTL is being applied. */
725 | @Test
726 | def testMinVersionsWithTTL(): Unit = {
727 | val ttl = 3600 // 1h TTL
728 |
729 | val desc = new HTableDescriptor("table")
730 | desc.addFamily(new HColumnDescriptor("family")
731 | .setMaxVersions(HConstants.ALL_VERSIONS) // retain all versions
732 | .setTimeToLive(ttl) // 1h TTL
733 | .setMinVersions(2) // retain at least 2 versions wrt TTL
734 | )
735 | val table = new FakeHTable(
736 | name = "table",
737 | conf = HBaseConfiguration.create(),
738 | desc = desc
739 | )
740 |
741 | val nowMS = System.currentTimeMillis
742 | val minTimestamp = nowMS - (ttl * 1000)
743 |
744 | // Write cells older than TTL, only 2 of these cells should be retained:
745 | for (index <- 0 until 9) {
746 | val timestamp = minTimestamp - index // absolutely older than TTL allows
747 | table.put(new Put("row").add("family", "q", timestamp, "value-%d".format(index)))
748 | }
749 |
750 | {
751 | val result = table.get(new Get("row").addFamily("family").setMaxVersions())
752 | val kvs = result.getColumn("family", "q")
753 | Assert.assertEquals(2, kvs.size)
754 | Assert.assertEquals(minTimestamp, kvs.get(0).getTimestamp)
755 | Assert.assertEquals(minTimestamp - 1, kvs.get(1).getTimestamp)
756 | }
757 |
758 | // Write cells within TTL range, all these cells should be kept,
759 | // but the 2 cells older than TTL should disappear:
760 | for (index <- 0 until 9) {
761 | val timestamp = nowMS + index // within TTL range
762 | table.put(new Put("row").add("family", "q", timestamp, "value-%d".format(index)))
763 | }
764 |
765 | {
766 | val result = table.get(new Get("row").addFamily("family").setMaxVersions())
767 | val kvs = result.getColumn("family", "q")
768 | Assert.assertEquals(9, kvs.size)
769 | Assert.assertEquals(nowMS, kvs.get(8).getTimestamp)
770 | }
771 | }
772 |
773 | /** Test for the HTable.append() method. */
774 | @Test
775 | def testAppend(): Unit = {
776 | val desc = new HTableDescriptor("table")
777 | desc.addFamily(new HColumnDescriptor("family")
778 | .setMaxVersions(HConstants.ALL_VERSIONS) // retain all versions
779 | )
780 | val table = new FakeHTable(
781 | name = "table",
782 | conf = HBaseConfiguration.create(),
783 | desc = desc
784 | )
785 |
786 | val key = "row key"
787 |
788 | {
789 | val result = table.append(new Append(key).add("family", "qualifier", "value1"))
790 | Assert.assertEquals(1, result.size)
791 | Assert.assertEquals(1, result.getColumn("family", "qualifier").size)
792 | Assert.assertEquals(
793 | "value1",
794 | bytesToString(result.getColumnLatest("family", "qualifier").getValue))
795 | }
796 | {
797 | val result = table.append(new Append(key).add("family", "qualifier", "value2"))
798 | Assert.assertEquals(1, result.size)
799 | Assert.assertEquals(1, result.getColumn("family", "qualifier").size)
800 | Assert.assertEquals(
801 | "value1value2",
802 | bytesToString(result.getColumnLatest("family", "qualifier").getValue))
803 | }
804 | {
805 | val result = table.get(new Get(key)
806 | .setMaxVersions()
807 | .addColumn("family", "qualifier"))
808 | Assert.assertEquals(2, result.size)
809 | val kvs = result.getColumn("family", "qualifier")
810 | Assert.assertEquals(2, kvs.size)
811 | Assert.assertEquals("value1value2", bytesToString(kvs.get(0).getValue))
812 | Assert.assertEquals("value1", bytesToString(kvs.get(1).getValue))
813 | }
814 | }
815 |
816 | /** Makes sure the max TTL does not overflow due to 32 bits integer multiplication. */
817 | @Test
818 | def testMaxTTLOverflow(): Unit = {
819 | // The following TTL is nearly 25 days, in seconds,
820 | // and causes a 32 bits overflow when converted to ms:
821 | // 2147484 * 1000 = -2147483296
822 | // while:
823 | // 2147484 * 1000L = 2147484000
824 | //
825 | val ttl = 2147484
826 |
827 | val desc = new HTableDescriptor("table")
828 | desc.addFamily(new HColumnDescriptor("family")
829 | .setMaxVersions(HConstants.ALL_VERSIONS) // retain all versions
830 | .setTimeToLive(2147484) // retain cells for ~25 days
831 | .setMinVersions(0) // no minimum number of versions to retain
832 | )
833 | val table = new FakeHTable(
834 | name = "table",
835 | conf = HBaseConfiguration.create(),
836 | desc = desc
837 | )
838 |
839 | // Writes a cell whose timestamp is now.
840 | // Since the TTL is ~25 days, the cell should not be discarded,
841 | // unless the 32 bits overflow occurs when converting the TTL to ms.
842 | // If the overflow occurs, the TTL becomes negative and all cells older than now + ~25 days
843 | // are discarded.
844 | val row = "row"
845 | val nowMS = System.currentTimeMillis
846 | table.put(new Put(row).add("family", "qualifier", nowMS, "value"))
847 |
848 | val result = table.get(new Get(row))
849 | Assert.assertEquals("value", Bytes.toString(result.getValue("family", "qualifier")))
850 | }
851 |
852 | // -----------------------------------------------------------------------------------------------
853 |
854 | /**
855 | * Returns the smallest row key strictly greater than the specified row.
856 | *
857 | * @param row HBase row key.
858 | * @return the smallest row strictly greater than the specified row.
859 | */
860 | private def nextRow(row: Array[Byte]): Array[Byte] = {
861 | return Arrays.copyOfRange(row, 0, row.size + 1)
862 | }
863 |
864 | /**
865 | * Populates a given table with some data.
866 | *
867 | * @param table HBase table to fill in.
868 | * @param count Number of rows, families, columns and versions to write.
869 | */
870 | private def populateTable(
871 | table: HTableInterface,
872 | count: Int = 4
873 | ): Unit = {
874 | for (rId <- 0 until count) {
875 | val rowKey = "key%d".format(rId)
876 | for (fId <- 0 until count) {
877 | val family = "family%d".format(fId)
878 | for (cId <- 0 until count) {
879 | val qualifier = "qualifier%d".format(cId)
880 | for (timestamp <- 0L until count) {
881 | table.put(new Put(rowKey)
882 | .add(family, qualifier, timestamp, "value%d".format(timestamp)))
883 | }
884 | }
885 | }
886 | }
887 | }
888 | }
889 |
--------------------------------------------------------------------------------
/src/main/scala/org/kiji/testing/fakehtable/FakeHTable.scala:
--------------------------------------------------------------------------------
1 | /**
2 | * (c) Copyright 2012 WibiData, Inc.
3 | *
4 | * See the NOTICE file distributed with this work for additional
5 | * information regarding copyright ownership.
6 | *
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | */
19 |
20 | package org.kiji.testing.fakehtable
21 |
22 | import java.io.PrintStream
23 | import java.lang.{Boolean => JBoolean}
24 | import java.lang.{Long => JLong}
25 | import java.util.{ArrayList => JArrayList}
26 | import java.util.Arrays
27 | import java.util.{Iterator => JIterator}
28 | import java.util.{List => JList}
29 | import java.util.{Map => JMap}
30 | import java.util.NavigableMap
31 | import java.util.NavigableSet
32 | import java.util.{TreeMap => JTreeMap}
33 | import java.util.{TreeSet => JTreeSet}
34 |
35 | import scala.Option.option2Iterable
36 | import scala.collection.JavaConverters.asScalaBufferConverter
37 | import scala.collection.JavaConverters.asScalaSetConverter
38 | import scala.collection.JavaConverters.mapAsScalaMapConverter
39 | import scala.collection.mutable.Buffer
40 | import scala.util.control.Breaks.break
41 | import scala.util.control.Breaks.breakable
42 |
43 | import org.apache.hadoop.conf.Configuration
44 | import org.apache.hadoop.hbase.HBaseConfiguration
45 | import org.apache.hadoop.hbase.Cell
46 | import org.apache.hadoop.hbase.CellUtil
47 | import org.apache.hadoop.hbase.client.Row
48 | import org.apache.hadoop.hbase.HColumnDescriptor
49 | import org.apache.hadoop.hbase.HConstants
50 | import org.apache.hadoop.hbase.HRegionInfo
51 | import org.apache.hadoop.hbase.HRegionLocation
52 | import org.apache.hadoop.hbase.HTableDescriptor
53 | import org.apache.hadoop.hbase.KeyValue
54 | import org.apache.hadoop.hbase.ServerName
55 | import org.apache.hadoop.hbase.TableName
56 | import org.apache.hadoop.hbase.client.Append
57 | import org.apache.hadoop.hbase.client.Delete
58 | import org.apache.hadoop.hbase.client.Durability
59 | import org.apache.hadoop.hbase.client.Get
60 | import org.apache.hadoop.hbase.client.HConnection
61 | import org.apache.hadoop.hbase.client.HTableInterface
62 | import org.apache.hadoop.hbase.client.Increment
63 | import org.apache.hadoop.hbase.client.Put
64 | import org.apache.hadoop.hbase.client.Result
65 | import org.apache.hadoop.hbase.client.ResultScanner
66 | import org.apache.hadoop.hbase.client.RowMutations
67 | import org.apache.hadoop.hbase.client.Scan
68 | import org.apache.hadoop.hbase.client.coprocessor.Batch
69 | import org.apache.hadoop.hbase.filter.Filter
70 | import org.apache.hadoop.hbase.protobuf.ProtobufUtil
71 | import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel
72 | import org.apache.hadoop.hbase.util.Bytes
73 | import org.kiji.testing.fakehtable.JNavigableMapWithAsScalaIterator.javaNavigableMapAsScalaIterator
74 | import org.slf4j.LoggerFactory
75 |
76 | /**
77 | * Fake in-memory HTable.
78 | *
79 | * @param name is the table name.
80 | * @param desc is the table HBase descriptor.
81 | * Optional and may currently be null (the fake HTable infers descriptors as needed).
82 | * @param conf is the HBase configuration.
83 | * @param autoFlush is the initial value for the table auto-flush property.
84 | * @param enabled is the initial state of the table.
85 | * @param autoFillDesc indicates whether descriptors are required or automatically filled-in.
86 | * @param hconnection Fake HConnection for this HTable.
87 | */
88 | class FakeHTable(
89 | val name: String,
90 | desc: HTableDescriptor,
91 | val conf: Configuration = HBaseConfiguration.create(),
92 | private var autoFlush: Boolean = false,
93 | private var writeBufferSize: Long = 1,
94 | var enabled: Boolean = true,
95 | autoFillDesc: Boolean = true,
96 | hconnection: FakeHConnection = new FakeHConnection(null)
97 | ) extends HTableInterface
98 | with FakeTypes {
99 | private val Log = LoggerFactory.getLogger(getClass)
100 | require(conf != null)
101 |
102 | /** Whether the table has been closed. */
103 | private var closed: Boolean = false
104 |
105 | /** A fake connection. */
106 | private val mFakeHConnection: FakeHConnection = hconnection
107 | private val mHConnection: HConnection =
108 | UntypedProxy.create(classOf[HConnection], mFakeHConnection)
109 |
110 | /** Region splits and locations. Protected by `this`. */
111 | private var regions: Seq[HRegionLocation] = Seq()
112 |
113 | /** Comparator for Bytes. */
114 | private final val BytesComparator: java.util.Comparator[Bytes] = Bytes.BYTES_COMPARATOR
115 |
116 | /** Map: row key -> family -> qualifier -> timestamp -> cell data. */
117 | private val rows: Table = new JTreeMap[Bytes, RowFamilies](BytesComparator)
118 |
119 | // -----------------------------------------------------------------------------------------------
120 |
121 | /** HBase descriptor of the table. */
122 | private val mDesc: HTableDescriptor = {
123 | desc match {
124 | case null => {
125 | require(autoFillDesc)
126 | new HTableDescriptor(name)
127 | }
128 | case desc => desc
129 | }
130 | }
131 |
132 | override def getTableName(): Array[Byte] = {
133 | return name.getBytes
134 | }
135 |
136 | override def getConfiguration(): Configuration = {
137 | return conf
138 | }
139 |
140 | override def getTableDescriptor(): HTableDescriptor = {
141 | return mDesc
142 | }
143 |
144 | override def exists(get: Get): Boolean = {
145 | return !this.get(get).isEmpty()
146 | }
147 |
148 | override def append(append: Append): Result = {
149 | /** Key values to return as a result. */
150 | val resultKVs = Buffer[KeyValue]()
151 |
152 | synchronized {
153 | val row = rows.get(append.getRow)
154 |
155 | /**
156 | * Gets the current value for the specified cell.
157 | *
158 | * @param kv contains the coordinate of the cell to report the current value of.
159 | * @return the current value of the specified cell, or null if the cell does not exist.
160 | */
161 | def getCurrentValue(kv: KeyValue): Bytes = {
162 | if (row == null) return null
163 | val family = row.get(kv.getFamily)
164 | if (family == null) return null
165 | val qualifier = family.get(kv.getQualifier)
166 | if (qualifier == null) return null
167 | val entry = qualifier.firstEntry
168 | if (entry == null) return null
169 | return entry.getValue
170 | }
171 |
172 | /** Build a put request with the appended cells. */
173 | val put = new Put(append.getRow)
174 |
175 | val timestamp = System.currentTimeMillis
176 |
177 | for ((family, kvs) <- append.getFamilyMap.asScala) {
178 | for (kv <- kvs.asScala) {
179 | val currentValue: Bytes = getCurrentValue(kv)
180 | val newValue: Bytes = {
181 | if (currentValue == null) kv.getValue else (currentValue ++ kv.getValue)
182 | }
183 | val appendedKV =
184 | new KeyValue(kv.getRow, kv.getFamily, kv.getQualifier, timestamp, newValue)
185 | put.add(appendedKV)
186 |
187 | if (append.isReturnResults) resultKVs += appendedKV.clone()
188 | }
189 | }
190 | this.put(put)
191 | }
192 | return new Result(resultKVs.toArray)
193 | }
194 |
195 | override def batch(actions: JList[_ <: Row], results: Array[Object]): Unit = {
196 | require(results.size == actions.size)
197 | val array = batch(actions)
198 | System.arraycopy(array, 0, results, 0, results.length)
199 | }
200 |
201 | override def batch(actions: JList[_ <: Row]): Array[Object] = {
202 | val results = Buffer[Object]()
203 | actions.asScala.foreach { action =>
204 | action match {
205 | case put: Put => {
206 | this.put(put)
207 | results += new Object()
208 | }
209 | case get: Get => {
210 | results += this.get(get)
211 | }
212 | case delete: Delete => {
213 | this.delete(delete)
214 | results += new Object()
215 | }
216 | case append: Append => {
217 | results += this.append(append)
218 | }
219 | case increment: Increment => {
220 | results += this.increment(increment)
221 | }
222 | case mutations: RowMutations => {
223 | this.mutateRow(mutations)
224 | }
225 | }
226 | }
227 | return results.toArray
228 | }
229 |
230 | override def get(get: Get): Result = {
231 | // get() could be built around scan(), to ensure consistent filters behavior.
232 | // For now, we use a shortcut:
233 | synchronized {
234 | val filter: Filter = getFilter(get.getFilter)
235 | filter.reset()
236 | if (filter.filterAllRemaining()) {
237 | return new Result()
238 | }
239 | val rowKey = get.getRow
240 | if (filter.filterRowKey(rowKey, 0, rowKey.size)) {
241 | return new Result()
242 | }
243 | val row = rows.get(rowKey)
244 | if (row == null) {
245 | return new Result()
246 | }
247 | val result = ProcessRow.makeResult(
248 | table = this,
249 | rowKey = rowKey,
250 | row = row,
251 | familyMap = getFamilyMapRequest(get.getFamilyMap),
252 | timeRange = get.getTimeRange,
253 | maxVersions = get.getMaxVersions,
254 | filter = filter
255 | )
256 | if (filter.filterRow()) {
257 | return new Result()
258 | }
259 | return result
260 | }
261 | }
262 |
263 | override def get(gets: JList[Get]): Array[Result] = {
264 | return gets.asScala.map(this.get(_)).toArray
265 | }
266 |
267 | @deprecated(message = "Deprecated method will not be implemented", since = "HBase 0.92")
268 | override def getRowOrBefore(row: Bytes, family: Bytes): Result = {
269 | sys.error("Deprecated method will not be implemented")
270 | }
271 |
272 | override def getScanner(scan: Scan): ResultScanner = {
273 | return new FakeResultScanner(scan)
274 | }
275 |
276 | override def getScanner(family: Bytes): ResultScanner = {
277 | return getScanner(new Scan().addFamily(family))
278 | }
279 |
280 | override def getScanner(family: Bytes, qualifier: Bytes): ResultScanner = {
281 | return getScanner(new Scan().addColumn(family, qualifier))
282 | }
283 |
284 | override def put(put: Put): Unit = {
285 | synchronized {
286 | val nowMS = System.currentTimeMillis
287 |
288 | val rowKey = put.getRow
289 | val rowFamilyMap = rows.asScala
290 | .getOrElseUpdate(rowKey, new JTreeMap[Bytes, FamilyQualifiers](BytesComparator))
291 | for ((family, kvs) <- put.getFamilyMap.asScala) {
292 | /** Map: qualifier -> time series. */
293 | val rowQualifierMap = rowFamilyMap.asScala
294 | .getOrElseUpdate(family, new JTreeMap[Bytes, ColumnSeries](BytesComparator))
295 |
296 | for (kv <- kvs.asScala) {
297 | require(Arrays.equals(family, kv.getFamily))
298 |
299 | /** Map: timestamp -> value. */
300 | val column = rowQualifierMap.asScala
301 | .getOrElseUpdate(kv.getQualifier, new JTreeMap[JLong, Bytes](TimestampComparator))
302 |
303 | val timestamp = {
304 | if (kv.getTimestamp == HConstants.LATEST_TIMESTAMP) {
305 | nowMS
306 | } else {
307 | kv.getTimestamp
308 | }
309 | }
310 |
311 | column.put(timestamp, kv.getValue)
312 | }
313 | }
314 | }
315 | }
316 |
317 | override def put(put: JList[Put]): Unit = {
318 | put.asScala.foreach(this.put(_))
319 | }
320 |
321 | /**
322 | * Checks the value of a cell. Caller must synchronize before calling.
323 | *
324 | * @param row Row key.
325 | * @param family Family.
326 | * @param qualifier Qualifier.
327 | * @param value Value to compare against.
328 | * Null means check that the specified cell does not exist.
329 | * @return whether the HBase cell check is successful, ie. whether the specified cell exists
330 | * and contains the given value, or whether the cell does not exist.
331 | */
332 | private def checkCell(row: Bytes, family: Bytes, qualifier: Bytes, value: Bytes): Boolean = {
333 | if (value == null) {
334 | val fmap = rows.get(row)
335 | if (fmap == null) return true
336 | val qmap = fmap.get(family)
337 | if (qmap == null) return true
338 | val tmap = qmap.get(qualifier)
339 | return (tmap == null) || tmap.isEmpty
340 | } else {
341 | val fmap = rows.get(row)
342 | if (fmap == null) return false
343 | val qmap = fmap.get(family)
344 | if (qmap == null) return false
345 | val tmap = qmap.get(qualifier)
346 | if ((tmap == null) || tmap.isEmpty) return false
347 | return Arrays.equals(tmap.firstEntry.getValue, value)
348 | }
349 | }
350 |
351 | override def checkAndPut(
352 | row: Bytes,
353 | family: Bytes,
354 | qualifier: Bytes,
355 | value: Bytes,
356 | put: Put
357 | ): Boolean = {
358 | synchronized {
359 | if (checkCell(row = row, family = family, qualifier = qualifier, value = value)) {
360 | this.put(put)
361 | return true
362 | } else {
363 | return false
364 | }
365 | }
366 | }
367 |
368 | /**
369 | * Removes empty maps for a specified row, family and/or qualifier. Caller must
370 | * synchronize before calling.
371 | *
372 | * @param rowKey Key of the row to clean up.
373 | * @param family Optional family to clean up. None means clean all families.
374 | * @param qualifier Optional qualifier to clean up. None means clean all qualifiers.
375 | */
376 | private def cleanupRow(rowKey: Bytes, family: Option[Bytes], qualifier: Option[Bytes]): Unit = {
377 | val row = rows.get(rowKey)
378 | if (row == null) { return }
379 |
380 | val families : Iterable[Bytes] = family match {
381 | case Some(_) => family
382 | case None => row.keySet.asScala
383 | }
384 | val emptyFamilies = Buffer[Bytes]() // empty families to clean up
385 | for (family <- families) {
386 | val rowQualifierMap = row.get(family)
387 | if (rowQualifierMap != null) {
388 | val qualifiers : Iterable[Bytes] = qualifier match {
389 | case Some(_) => qualifier
390 | case None => rowQualifierMap.keySet.asScala
391 | }
392 | val emptyQualifiers = Buffer[Bytes]()
393 | for (qualifier <- qualifiers) {
394 | val timeSeries = rowQualifierMap.get(qualifier)
395 | if ((timeSeries != null) && timeSeries.isEmpty) {
396 | emptyQualifiers += qualifier
397 | }
398 | }
399 | emptyQualifiers.foreach { qualifier => rowQualifierMap.remove(qualifier) }
400 |
401 | if (rowQualifierMap.isEmpty) {
402 | emptyFamilies += family
403 | }
404 | }
405 | }
406 | emptyFamilies.foreach { family => row.remove(family) }
407 | if (row.isEmpty) {
408 | rows.remove(rowKey)
409 | }
410 | }
411 |
412 | override def delete(delete: Delete): Unit = {
413 | synchronized {
414 | val rowKey = delete.getRow
415 | val row = rows.get(rowKey)
416 | if (row == null) { return }
417 |
418 | if (delete.getFamilyMap.isEmpty) {
419 | for ((family, qualifiers) <- row.asScala) {
420 | for ((qualifier, series) <- qualifiers.asScala) {
421 | series.subMap(delete.getTimeStamp, true, 0, true).clear()
422 | }
423 | }
424 | cleanupRow(rowKey = rowKey, family = None, qualifier = None)
425 | return
426 | }
427 |
428 | for ((requestedFamily, kvs) <- delete.getFamilyMap.asScala) {
429 | val rowQualifierMap = row.get(requestedFamily)
430 | if (rowQualifierMap != null) {
431 | for (kv <- kvs.asScala) {
432 | require(kv.isDelete)
433 | if (kv.isDeleteFamily) {
434 | // Removes versions of an entire family prior to the specified timestamp:
435 | for ((qualifier, series) <- rowQualifierMap.asScala) {
436 | series.subMap(kv.getTimestamp, true, 0, true).clear()
437 | }
438 | } else if (kv.isDeleteColumnOrFamily) {
439 | // Removes versions of a column prior to the specified timestamp:
440 | val series = rowQualifierMap.get(kv.getQualifier)
441 | if (series != null) {
442 | series.subMap(kv.getTimestamp, true, 0, true).clear()
443 | }
444 | } else {
445 | // Removes exactly one cell:
446 | val series = rowQualifierMap.get(kv.getQualifier)
447 | if (series != null) {
448 | val timestamp = {
449 | if (kv.getTimestamp == HConstants.LATEST_TIMESTAMP) {
450 | series.firstKey
451 | } else {
452 | kv.getTimestamp
453 | }
454 | }
455 | series.remove(timestamp)
456 | }
457 | }
458 | }
459 | }
460 | cleanupRow(rowKey = rowKey, family = Some(requestedFamily), qualifier = None)
461 | }
462 | }
463 | }
464 |
465 | override def delete(deletes: JList[Delete]): Unit = {
466 | deletes.asScala.foreach(this.delete(_))
467 | }
468 |
469 | override def checkAndDelete(
470 | row: Bytes,
471 | family: Bytes,
472 | qualifier: Bytes,
473 | value: Bytes,
474 | delete: Delete
475 | ): Boolean = {
476 | synchronized {
477 | if (checkCell(row = row, family = family, qualifier = qualifier, value = value)) {
478 | this.delete(delete)
479 | return true
480 | } else {
481 | return false
482 | }
483 | }
484 | }
485 |
486 | override def increment(increment: Increment): Result = {
487 | synchronized {
488 | val nowMS = System.currentTimeMillis
489 |
490 | val rowKey = increment.getRow
491 | val row = rows.asScala
492 | .getOrElseUpdate(rowKey, new JTreeMap[Bytes, FamilyQualifiers](BytesComparator))
493 | val familyMap = new JTreeMap[Bytes, NavigableSet[Bytes]](BytesComparator)
494 |
495 | for ((family: Array[Byte], qualifierMap: JList[Cell]) <- increment.getFamilyCellMap.asScala) {
496 | val qualifierSet = familyMap.asScala
497 | .getOrElseUpdate(family, new JTreeSet[Bytes](BytesComparator))
498 | val rowQualifierMap = row.asScala
499 | .getOrElseUpdate(family, new JTreeMap[Bytes, ColumnSeries](BytesComparator))
500 |
501 | for (cell: Cell <- qualifierMap.asScala) {
502 | val qualifier = CellUtil.cloneQualifier(cell)
503 | val amount = Bytes.toLong(CellUtil.cloneValue(cell))
504 | qualifierSet.add(qualifier)
505 | val rowTimeSeries = rowQualifierMap.asScala
506 | .getOrElseUpdate(qualifier, new JTreeMap[JLong, Bytes](TimestampComparator))
507 | val currentCounter = {
508 | if (rowTimeSeries.isEmpty) {
509 | 0
510 | } else {
511 | Bytes.toLong(rowTimeSeries.firstEntry.getValue)
512 | }
513 | }
514 | val newCounter = currentCounter + amount
515 | Log.debug("Updating counter from %d to %d".format(currentCounter, newCounter))
516 | rowTimeSeries.put(nowMS, Bytes.toBytes(newCounter))
517 | }
518 | }
519 |
520 | return ProcessRow.makeResult(
521 | table = this,
522 | rowKey = increment.getRow,
523 | row = row,
524 | familyMap = familyMap,
525 | timeRange = increment.getTimeRange,
526 | maxVersions = 1
527 | )
528 | }
529 | }
530 |
531 | override def incrementColumnValue(
532 | row: Bytes,
533 | family: Bytes,
534 | qualifier: Bytes,
535 | amount: Long
536 | ): Long = {
537 | return this.incrementColumnValue(row, family, qualifier, amount, writeToWAL = true)
538 | }
539 |
540 | override def incrementColumnValue(
541 | row: Bytes,
542 | family: Bytes,
543 | qualifier: Bytes,
544 | amount: Long,
545 | writeToWAL: Boolean
546 | ): Long = {
547 | val inc = new Increment(row)
548 | .addColumn(family, qualifier, amount)
549 | val result = this.increment(inc)
550 | require(!result.isEmpty)
551 | return Bytes.toLong(result.getValue(family, qualifier))
552 | }
553 |
554 | override def mutateRow(mutations: RowMutations): Unit = {
555 | synchronized {
556 | for (mutation <- mutations.getMutations.asScala) {
557 | mutation match {
558 | case put: Put => this.put(put)
559 | case delete: Delete => this.delete(delete)
560 | case _ => sys.error("Unexpected row mutation: " + mutation)
561 | }
562 | }
563 | }
564 | }
565 |
566 | override def setAutoFlush(autoFlush: Boolean, clearBufferOnFail: Boolean): Unit = {
567 | synchronized {
568 | this.autoFlush = autoFlush
569 | // Ignore clearBufferOnFail
570 | }
571 | }
572 |
573 | override def setAutoFlush(autoFlush: Boolean): Unit = {
574 | synchronized {
575 | this.autoFlush = autoFlush
576 | }
577 | }
578 |
579 | override def isAutoFlush(): Boolean = {
580 | synchronized {
581 | return autoFlush
582 | }
583 | }
584 |
585 | override def flushCommits(): Unit = {
586 | // Do nothing
587 | }
588 |
589 | override def setWriteBufferSize(writeBufferSize: Long): Unit = {
590 | synchronized {
591 | this.writeBufferSize = writeBufferSize
592 | }
593 | }
594 |
595 | override def getWriteBufferSize(): Long = {
596 | synchronized {
597 | return writeBufferSize
598 | }
599 | }
600 |
601 | override def close(): Unit = {
602 | synchronized {
603 | this.closed = true
604 | }
605 | }
606 |
607 | // -----------------------------------------------------------------------------------------------
608 |
609 | /** @return the regions info for this table. */
610 | private[fakehtable] def getRegions(): JList[HRegionInfo] = {
611 | val list = new java.util.ArrayList[HRegionInfo]()
612 | synchronized {
613 | for (region <- regions) {
614 | list.add(region.getRegionInfo)
615 | }
616 | }
617 | return list
618 | }
619 |
620 | /**
621 | * Converts a list of region boundaries (null excluded) into a stream of regions.
622 | *
623 | * @param split Region boundaries, first and last null/empty excluded.
624 | * @return a stream of (start, end) regions.
625 | */
626 | private def toRegions(split: Seq[Bytes]): Iterator[(Bytes, Bytes)] = {
627 | if (!split.isEmpty) {
628 | require((split.head != null) && !split.head.isEmpty)
629 | require((split.last != null) && !split.last.isEmpty)
630 | }
631 | val startKeys = Iterator(null) ++ split.iterator
632 | val endKeys = split.iterator ++ Iterator(null)
633 | return startKeys.zip(endKeys).toIterator
634 | }
635 |
636 | /**
637 | * Sets the region splits for this table.
638 | *
639 | * @param split Split boundaries (excluding the first and last null/empty).
640 | */
641 | private[fakehtable] def setSplit(split: Array[Bytes]): Unit = {
642 | val fakePort = 1234
643 | val tableName: Bytes = Bytes.toBytes(name)
644 |
645 | val newRegions = Buffer[HRegionLocation]()
646 | for ((start, end) <- toRegions(split)) {
647 | val fakeHost = "fake-location-%d".format(newRegions.size)
648 | val regionInfo = new HRegionInfo(TableName.valueOf(tableName), start, end)
649 | val seqNum = System.currentTimeMillis()
650 | newRegions += new HRegionLocation(
651 | regionInfo,
652 | ServerName.valueOf(fakeHost, fakePort, /* startCode = */ 0),
653 | /* seqNum = */ 0
654 | )
655 | }
656 | synchronized {
657 | this.regions = newRegions.toSeq
658 | }
659 | }
660 |
661 | // -----------------------------------------------------------------------------------------------
662 | // HTable methods that are not part of HTableInterface, but required anyway:
663 |
664 | /** See HTable.getRegionLocation(). */
665 | def getRegionLocation(row: String): HRegionLocation = {
666 | return getRegionLocation(Bytes.toBytes(row))
667 | }
668 |
669 | /** See HTable.getRegionLocation(). */
670 | def getRegionLocation(row: Bytes): HRegionLocation = {
671 | return getRegionLocation(row, false)
672 | }
673 |
674 | /** See HTable.getRegionLocation(). */
675 | def getRegionLocation(row: Bytes, reload: Boolean): HRegionLocation = {
676 | synchronized {
677 | for (region <- regions) {
678 | val start = region.getRegionInfo.getStartKey
679 | val end = region.getRegionInfo.getEndKey
680 | // start ≤ row < end:
681 | if ((Bytes.compareTo(start, row) <= 0)
682 | && (end.isEmpty || (Bytes.compareTo(row, end) < 0))) {
683 | return region
684 | }
685 | }
686 | }
687 | sys.error("Invalid region split: last region must does not end with empty row key")
688 | }
689 |
690 | /** See HTable.getRegionLocations(). */
691 | def getRegionLocations(): NavigableMap[HRegionInfo, ServerName] = {
692 | val map = new JTreeMap[HRegionInfo, ServerName]()
693 | synchronized {
694 | for (region <- regions) {
695 | map.put(region.getRegionInfo, new ServerName(region.getHostname, region.getPort, 0))
696 | }
697 | }
698 | return map
699 | }
700 |
701 | /**
702 | * See HTable.getRegionsInRange(startKey, endKey).
703 | *
704 | * Adapted from org.apache.hadoop.hbase.client.HTable.
705 | * Note: if startKey == endKey, this returns the location for startKey.
706 | */
707 | def getRegionsInRange(startKey: Bytes, endKey: Bytes): JList[HRegionLocation] = {
708 | val endKeyIsEndOfTable = Bytes.equals(endKey, HConstants.EMPTY_END_ROW)
709 | if ((Bytes.compareTo(startKey, endKey) > 0) && !endKeyIsEndOfTable) {
710 | throw new IllegalArgumentException("Invalid range: %s > %s".format(
711 | Bytes.toStringBinary(startKey), Bytes.toStringBinary(endKey)))
712 | }
713 | val regionList = new JArrayList[HRegionLocation]()
714 | var currentKey = startKey
715 | do {
716 | val regionLocation = getRegionLocation(currentKey, false)
717 | regionList.add(regionLocation)
718 | currentKey = regionLocation.getRegionInfo().getEndKey()
719 | } while (!Bytes.equals(currentKey, HConstants.EMPTY_END_ROW)
720 | && (endKeyIsEndOfTable || Bytes.compareTo(currentKey, endKey) < 0))
721 | return regionList
722 | }
723 |
724 | /** See HTable.getConnection(). */
725 | def getConnection(): HConnection = {
726 | mHConnection
727 | }
728 |
729 | // -----------------------------------------------------------------------------------------------
730 |
731 | def toHex(bytes: Bytes): String = {
732 | return bytes.iterator
733 | .map { byte => "%02x".format(byte) }
734 | .mkString(":")
735 | }
736 |
737 | def toString(bytes: Bytes): String = {
738 | return Bytes.toStringBinary(bytes)
739 | }
740 |
741 | /**
742 | * Dumps the content of the fake HTable.
743 | *
744 | * @param out Optional print stream to write to.
745 | */
746 | def dump(out: PrintStream = Console.out): Unit = {
747 | synchronized {
748 | for ((rowKey, familyMap) <- rows.asScalaIterator) {
749 | for ((family, qualifierMap) <- familyMap.asScalaIterator) {
750 | for ((qualifier, timeSeries) <- qualifierMap.asScalaIterator) {
751 | for ((timestamp, value) <- timeSeries.asScalaIterator) {
752 | out.println("row=%s family=%s qualifier=%s timestamp=%d value=%s".format(
753 | toString(rowKey),
754 | toString(family),
755 | toString(qualifier),
756 | timestamp,
757 | toHex(value)))
758 | }
759 | }
760 | }
761 | }
762 | }
763 | }
764 |
765 | /**
766 | * Instantiates the specified HBase filter.
767 | *
768 | * @param filterSpec HBase filter specification, or null for no filter.
769 | * @return a new instance of the specified filter.
770 | */
771 | private def getFilter(filterSpec: Filter): Filter = {
772 | Option(filterSpec) match {
773 | case Some(hfilter) => ProtobufUtil.toFilter(ProtobufUtil.toFilter(hfilter))
774 | case None => PassThroughFilter
775 | }
776 | }
777 |
778 | /**
779 | * Ensures a family map from a Get/Scan is sorted.
780 | *
781 | * @param request Requested family/qualifier map.
782 | * @return The requested family/qualifiers, as a sorted map.
783 | */
784 | private def getFamilyMapRequest(
785 | request: JMap[Bytes, NavigableSet[Bytes]]
786 | ): NavigableMap[Bytes, NavigableSet[Bytes]] = {
787 | if (request.isInstanceOf[NavigableMap[_, _]]) {
788 | return request.asInstanceOf[NavigableMap[Bytes, NavigableSet[Bytes]]]
789 | }
790 | val map = new JTreeMap[Bytes, NavigableSet[Bytes]](BytesComparator)
791 | for ((family, qualifiers) <- request.asScala) {
792 | map.put(family, qualifiers)
793 | }
794 | return map
795 | }
796 |
797 | /**
798 | * Reports the HBase descriptor for a column family.
799 | *
800 | * @param family is the column family to report the descriptor of.
801 | * @return the descriptor of the specified column family.
802 | */
803 | private[fakehtable] def getFamilyDesc(family: Bytes): HColumnDescriptor = {
804 | val desc = getTableDescriptor()
805 | val familyDesc = desc.getFamily(family)
806 | if (familyDesc != null) {
807 | return familyDesc
808 | }
809 | require(autoFillDesc)
810 | val newFamilyDesc = new HColumnDescriptor(family)
811 | // Note on default parameters:
812 | // - min versions is 0
813 | // - max versions is 3
814 | // - TTL is forever
815 | desc.addFamily(newFamilyDesc)
816 | return newFamilyDesc
817 | }
818 |
819 | // -----------------------------------------------------------------------------------------------
820 |
821 | /**
822 | * ResultScanner for a fake in-memory HTable.
823 | *
824 | * @param scan Scan options.
825 | */
826 | private class FakeResultScanner(
827 | val scan: Scan
828 | ) extends ResultScanner with JIterator[Result] {
829 |
830 | /** Requested family/qualifiers map. */
831 | private val requestedFamilyMap: NavigableMap[Bytes, NavigableSet[Bytes]] =
832 | getFamilyMapRequest(scan.getFamilyMap)
833 |
834 | /** Key of the row to return on the next call to next(). Null means no more row. */
835 | private var key: Bytes = {
836 | synchronized {
837 | if (rows.isEmpty) {
838 | null
839 | } else if (scan.getStartRow.isEmpty) {
840 | rows.firstKey
841 | } else {
842 | rows.ceilingKey(scan.getStartRow)
843 | }
844 | }
845 | }
846 | if (!scan.getStopRow.isEmpty
847 | && (key == null || BytesComparator.compare(key, scan.getStopRow) >= 0)) {
848 | key = null
849 | }
850 |
851 | /** HBase row/column filter. */
852 | val filter = getFilter(scan.getFilter)
853 |
854 | /** Next result to return. */
855 | private var nextResult: Result = getNextResult()
856 |
857 | override def hasNext(): Boolean = {
858 | return (nextResult != null)
859 | }
860 |
861 | override def next(): Result = {
862 | val result = nextResult
863 | nextResult = getNextResult()
864 | return result
865 | }
866 |
867 | /** @return the next non-empty result. */
868 | private def getNextResult(): Result = {
869 | while (true) {
870 | getResultForNextRow() match {
871 | case None => return null
872 | case Some(result) => {
873 | if (!result.isEmpty) {
874 | return result
875 | }
876 | }
877 | }
878 | }
879 | // next() returns when a non empty Result is found or when there are no more rows:
880 | sys.error("dead code")
881 | }
882 |
883 | /**
884 | * @return the next row key, or null if there is no more row. Caller must synchronize
885 | * on `FakeHTable.this`.
886 | */
887 | private def nextRowKey(): Bytes = {
888 | if (key == null) { return null }
889 | val rowKey = key
890 | key = rows.higherKey(rowKey)
891 | if ((key != null)
892 | && !scan.getStopRow.isEmpty
893 | && (BytesComparator.compare(key, scan.getStopRow) >= 0)) {
894 | key = null
895 | }
896 | return rowKey
897 | }
898 |
899 | /** @return a Result, potentially empty, for the next row. */
900 | private def getResultForNextRow(): Option[Result] = {
901 | FakeHTable.this.synchronized {
902 | filter.reset()
903 | if (filter.filterAllRemaining) { return None }
904 |
905 | val rowKey = nextRowKey()
906 | if (rowKey == null) { return None }
907 | if (filter.filterRowKey(rowKey, 0, rowKey.size)) {
908 | // Row is filtered out based on its key, return an empty Result:
909 | return Some(new Result())
910 | }
911 |
912 | /** Map: family -> qualifier -> time stamp -> cell value */
913 | val row = rows.get(rowKey)
914 | require(row != null)
915 |
916 | val result = ProcessRow.makeResult(
917 | table = FakeHTable.this,
918 | rowKey = rowKey,
919 | row = row,
920 | familyMap = requestedFamilyMap,
921 | timeRange = scan.getTimeRange,
922 | maxVersions = scan.getMaxVersions,
923 | filter = filter
924 | )
925 | if (filter.filterRow()) {
926 | // Filter finally decided to exclude the row, return an empty Result:
927 | return Some(new Result())
928 | }
929 | return Some(result)
930 | }
931 | }
932 |
933 | override def next(nrows: Int): Array[Result] = {
934 | val results = Buffer[Result]()
935 | breakable {
936 | for (nrow <- 0 until nrows) {
937 | next() match {
938 | case null => break
939 | case row => results += row
940 | }
941 | }
942 | }
943 | return results.toArray
944 | }
945 |
946 | override def close(): Unit = {
947 | // Nothing to close
948 | }
949 |
950 | override def iterator(): JIterator[Result] = {
951 | return this
952 | }
953 |
954 | override def remove(): Unit = {
955 | throw new UnsupportedOperationException
956 | }
957 | }
958 |
959 | override def batchCallback[R](
960 | x$1: JList[_ <: Row],
961 | x$2: Batch.Callback[R]
962 | ): Array[Object] = {
963 | sys.error("Not implemented")
964 | }
965 |
966 | override def batchCallback[R](
967 | x$1: java.util.List[_ <: Row],
968 | x$2: Array[Object],
969 | x$3: Batch.Callback[R]
970 | ): Unit = { sys.error("Not implemented") }
971 |
972 | override def coprocessorService[T <: com.google.protobuf.Service, R](
973 | x$1: Class[T],
974 | x$2: Array[Byte],
975 | x$3: Array[Byte],
976 | x$4: Batch.Call[T, R],
977 | x$5: Batch.Callback[R]
978 | ):Unit = {
979 | sys.error("Not implemented")
980 | }
981 |
982 | override def coprocessorService[T <: com.google.protobuf.Service, R](
983 | x$1: Class[T],
984 | x$2: Array[Byte],
985 | x$3: Array[Byte],
986 | x$4: Batch.Call[T, R]
987 | ):java.util.Map[Array[Byte],R] = {
988 | sys.error("Not implemented")
989 | }
990 |
991 | override def coprocessorService(x$1: Array[Byte]): CoprocessorRpcChannel = {
992 | sys.error("Not implemented")
993 | }
994 |
995 | override def exists(gets: JList[Get]): Array[JBoolean] = {
996 | val exists: Array[JBoolean] = new Array[JBoolean](gets.size)
997 | synchronized {
998 | for (index <- 0 until gets.size) {
999 | exists(index) = this.exists(gets.get(index))
1000 | }
1001 | }
1002 | exists
1003 | }
1004 |
1005 | override def getName(): TableName = {
1006 | TableName.valueOf(mDesc.getName)
1007 | }
1008 |
1009 | override def incrementColumnValue(
1010 | x$1: Array[Byte],
1011 | x$2: Array[Byte],
1012 | x$3: Array[Byte],
1013 | x$4: Long,
1014 | x$5: Durability
1015 | ): Long = {
1016 | sys.error("Not implemented")
1017 | }
1018 |
1019 | override def setAutoFlushTo(x$1: Boolean):Unit = {
1020 | sys.error("Not implemented")
1021 | }
1022 |
1023 | // -----------------------------------------------------------------------------------------------
1024 |
1025 | }
1026 |
--------------------------------------------------------------------------------